// Filters overlapping bounding boxes with lower probabilities.
private float IntersectionOverUnion(UnityEngine.Rect boundingBoxA, UnityEngine.Rect boundingBoxB)
{
var areaA = boundingBoxA.width * boundingBoxA.height;
if (areaA <= 0)
{
return(0);
}
var areaB = boundingBoxB.width * boundingBoxB.height;
if (areaB <= 0)
{
return(0);
}
var minX = Math.Max(boundingBoxA.xMin, boundingBoxB.xMin);
var minY = Math.Max(boundingBoxA.yMin, boundingBoxB.yMin);
var maxX = Math.Min(boundingBoxA.xMax, boundingBoxB.xMax);
var maxY = Math.Min(boundingBoxA.yMax, boundingBoxB.yMax);
var intersectionArea = Math.Max(maxY - minY, 0) * Math.Max(maxX - minX, 0);
return(intersectionArea / (areaA + areaB - intersectionArea));
}
private void DetectInRegion(Mat img, Rect region, List <Rect> detectedObjectsInRegions, FaceLandmarkDetector landmarkDetector)
{
Rect r0 = new Rect(new Point(), img.size());
Rect r1 = new Rect(region.x, region.y, region.width, region.height);
Rect.inflate(r1, (int)((r1.width * coeffTrackingWindowSize) - r1.width) / 2,
(int)((r1.height * coeffTrackingWindowSize) - r1.height) / 2);
r1 = Rect.intersect(r0, r1);
if ((r1.width <= 0) || (r1.height <= 0))
{
Debug.Log("detectInRegion: Empty intersection");
return;
}
using (Mat img1_roi = new Mat(img, r1))
using (Mat img1 = new Mat(r1.size(), img.type()))
{
img1_roi.copyTo(img1);
OpenCVForUnityUtils.SetImage(landmarkDetector, img1);
List <UnityEngine.Rect> detectResult = landmarkDetector.Detect();
int len = detectResult.Count;
for (int i = 0; i < len; i++)
{
UnityEngine.Rect tmp = detectResult[i];
Rect r = new Rect((int)(tmp.x + r1.x), (int)(tmp.y + r1.y), (int)tmp.width, (int)tmp.height);
detectedObjectsInRegions.Add(r);
}
}
}
private UnityEngine.Rect DetectFace(Mat mat)
{
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, mat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
if (result.Count >= 1)
{
return(result[0]);
}
}
else
{
using (Mat grayMat = new Mat())
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
// convert image to greyscale.
Imgproc.cvtColor(mat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
List <Rect> faceList = faces.toList();
if (faceList.Count >= 1)
{
UnityEngine.Rect r = new UnityEngine.Rect(faceList[0].x, faceList[0].y, faceList[0].width, faceList[0].height);
// corrects the deviation of a detection result between OpenCV and Dlib.
r.y += (int)(r.height * 0.1f);
return(r);
}
}
}
return(new UnityEngine.Rect());
}
private List <Vector2> DetectFaceLandmarkPoints(Mat mat, UnityEngine.Rect rect)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, mat);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
return(points);
}
/// <summary>
/// Raises the scan face mask button click event.
/// </summary>
public void OnScanFaceMaskButtonClick()
{
RemoveFaceMask();
// Capture webcam frame.
if (webCamTextureToMatHelper.IsPlaying())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
faceRectInMask = DetectFace(rgbaMat);
if (faceRectInMask.width == 0 && faceRectInMask.height == 0)
{
Debug.Log("A face could not be detected from the input image.");
return;
}
Rect rect = new Rect((int)faceRectInMask.x, (int)faceRectInMask.y, (int)faceRectInMask.width, (int)faceRectInMask.height);
rect.inflate(rect.x / 5, rect.y / 5);
rect = rect.intersect(new Rect(0, 0, rgbaMat.width(), rgbaMat.height()));
faceMaskTexture = new Texture2D(rect.width, rect.height, TextureFormat.RGBA32, false);
faceMaskMat = new Mat(rgbaMat, rect).clone();
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(faceMaskMat, faceMaskTexture);
Debug.Log("faceMaskMat ToString " + faceMaskMat.ToString());
faceRectInMask = DetectFace(faceMaskMat);
faceLandmarkPointsInMask = DetectFaceLandmarkPoints(faceMaskMat, faceRectInMask);
if (faceRectInMask.width == 0 && faceRectInMask.height == 0)
{
RemoveFaceMask();
Debug.Log("A face could not be detected from the input image.");
}
}
}
/// <summary>
/// Raises the change face mask button click event.
/// </summary>
public void OnChangeFaceMaskButtonClick()
{
RemoveFaceMask();
if (faceMaskDatas.Count == 0)
{
return;
}
FaceMaskData maskData = faceMaskDatas [faceMaskDataIndex];
faceMaskDataIndex = (faceMaskDataIndex < faceMaskDatas.Count - 1) ? faceMaskDataIndex + 1 : 0;
if (maskData == null)
{
Debug.LogError("maskData == null");
return;
}
if (maskData.image == null)
{
Debug.LogError("image == null");
return;
}
if (maskData.landmarkPoints.Count != 68)
{
Debug.LogError("landmarkPoints.Count != 68");
return;
}
faceMaskTexture = maskData.image;
faceMaskMat = new Mat(faceMaskTexture.height, faceMaskTexture.width, CvType.CV_8UC4);
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
if (maskData.isDynamicMode)
{
faceRectInMask = DetectFace(faceMaskMat);
faceLandmarkPointsInMask = DetectFaceLandmarkPoints(faceMaskMat, faceRectInMask);
maskData.faceRect = faceRectInMask;
maskData.landmarkPoints = faceLandmarkPointsInMask;
}
else
{
faceRectInMask = maskData.faceRect;
faceLandmarkPointsInMask = maskData.landmarkPoints;
}
if (faceRectInMask.width == 0 && faceRectInMask.height == 0)
{
RemoveFaceMask();
Debug.LogError("A face could not be detected from the input image.");
}
enableColorCorrectionToggle.isOn = maskData.enableColorCorrection;
}
/// <summary>
/// Raises the set face mask button click event.
/// </summary>
public void OnSetFaceMaskButtonClick()
{
if (faceMaskMat != null)
{
faceMaskMat.Dispose();
}
faceMaskTexture = Resources.Load("face_mask") as Texture2D;
faceMaskMat = new Mat(faceMaskTexture.height, faceMaskTexture.width, CvType.CV_8UC4);
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
Debug.Log("faceMaskMat ToString " + faceMaskMat.ToString());
detectedFaceRect = new UnityEngine.Rect();
}
private void DrawRects(Rect[] rects, float imageWidth, float imageHeight)
{
UnityEngine.Rect[] overlayRects = new UnityEngine.Rect[rects.Length];
for (int i = 0; i < rects.Length; i++)
{
overlayRects[i] = new UnityEngine.Rect(rects[i].x / imageWidth
, rects[i].y / imageHeight
, rects[i].width / imageWidth
, rects[i].height / imageHeight);
}
rectOverlay.DrawRects(overlayRects);
}
private void DetectObject(Mat img, out List <Rect> detectedObjects, FaceLandmarkDetector landmarkDetector)
{
OpenCVForUnityUtils.SetImage(landmarkDetector, img);
List <UnityEngine.Rect> detectResult = landmarkDetector.Detect();
detectedObjects = new List <Rect>();
int len = detectResult.Count;
for (int i = 0; i < len; i++)
{
UnityEngine.Rect r = detectResult[i];
detectedObjects.Add(new Rect((int)r.x, (int)r.y, (int)r.width, (int)r.height));
}
}
public FaceEmbeddings ExtractFaceEmbeddings(Texture2D imageTex, UnityEngine.Rect ROI)
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
var embedder = Dnn.readNetFromTorch(model_filepath);
Mat img = new Mat(imageTex.height, imageTex.width, CvType.CV_8UC3);
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Utils.texture2DToMat(imageTex, img);
if (img.empty())
{
Debug.LogError(input_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
Mat cropped_face = img.submat((int)ROI.y, (int)ROI.y + (int)ROI.height, (int)ROI.x, (int)ROI.width + (int)ROI.x);
Imgproc.cvtColor(cropped_face, cropped_face, Imgproc.COLOR_BGR2RGB);
var faceBlob = Dnn.blobFromImage(cropped_face, scalefactor, new Size(inpWidth, inpHeight), new Scalar(0, 0, 0), true, false);
embedder.setInput(faceBlob);
var netOut = embedder.forward();
var embeddings = new FaceEmbeddings(netOut, 128);
if (gameObject.GetComponent <Renderer>() != null && displayBB)
{
GenericUtils.AdjustImageScale(cropped_face, this.gameObject);
Texture2D texture = new Texture2D(cropped_face.cols(), cropped_face.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(cropped_face, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
}
embedder.Dispose();
cropped_face.Dispose();
img.Dispose();
netOut.Dispose();
return(embeddings);
}
public Rect ExtendBB(Mat img, UnityEngine.Rect rect, float scale_factor)
{
float x_new = rect.x - rect.width * scale_factor / 2;
float y_new = rect.y - rect.height * scale_factor / 2;
float width_new = rect.width * (1 + scale_factor);
float height_new = rect.height * (1 + scale_factor);
if (x_new < 0)
{
x_new = 1;
}
if (y_new < 0)
{
y_new = 1;
}
var xmax = x_new + width_new;
var ymax = y_new + height_new;
var img_width = img.size().width;
var img_height = img.size().height;
if (xmax > img_width)
{
width_new = rect.width;
x_new = rect.x;
}
if (ymax > img_height)
{
height_new = rect.height;
y_new = rect.y;
}
Rect ROI = new Rect((int)x_new, (int)y_new, (int)width_new, (int)height_new);
return(ROI);
}
public override void UpdateValue()
{
if (cascade == null)
{
return;
}
if (matSourceGetterInterface == null)
{
return;
}
didUpdateFaceRect = false;
Mat rgbaMat = (useDownScaleMat) ? matSourceGetterInterface.GetDownScaleMatSource() : matSourceGetterInterface.GetMatSource();
if (rgbaMat != null)
{
if (isDebugMode && screen != null)
{
if (debugMat != null && (debugMat.width() != rgbaMat.width() || debugMat.height() != rgbaMat.height()))
{
debugMat.Dispose();
debugMat = null;
}
debugMat = debugMat ?? new Mat(rgbaMat.rows(), rgbaMat.cols(), rgbaMat.type());
if (hideImage)
{
debugMat.setTo(new Scalar(0, 0, 0, 255));
}
else
{
rgbaMat.copyTo(debugMat);
}
if (debugTexture != null && (debugTexture.width != debugMat.width() || debugTexture.height != debugMat.height()))
{
Texture2D.Destroy(debugTexture);
debugTexture = null;
}
if (debugTexture == null)
{
debugTexture = new Texture2D(debugMat.width(), debugMat.height(), TextureFormat.RGBA32, false, false);
Vector2 size = screen.rectTransform.sizeDelta;
screen.rectTransform.sizeDelta = new Vector2(size.x, size.x * (float)debugMat.height() / (float)debugMat.width());
}
if (debugColors != null && debugColors.Length != debugMat.width() * debugMat.height())
{
debugColors = new Color32[debugMat.width() * debugMat.height()];
}
screen.texture = debugTexture;
screen.enabled = true;
}
else
{
if (screen != null)
{
screen.enabled = false;
}
}
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
Imgproc.equalizeHist(grayMat, grayMat);
if (cascade != null)
{
cascade.detectMultiScale(grayMat, faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE
new Size(grayMat.cols() * 0.2, grayMat.rows() * 0.2), new Size());
}
Rect[] rects = faces.toArray();
for (int i = 0; i < rects.Length; i++)
{
if (i == 0)
{
Rect r = rects [i];
if (useDownScaleMat)
{
// restore to original size rect
float downscaleRatio = matSourceGetterInterface.GetDownScaleRatio();
faceRect = new UnityEngine.Rect(
r.x * downscaleRatio,
r.y * downscaleRatio,
r.width * downscaleRatio,
r.height * downscaleRatio
);
}
else
{
faceRect = new UnityEngine.Rect(r.x, r.y, r.width, r.height);
}
didUpdateFaceRect = true;
//Debug.Log ("detect faces " + rects [i]);
if (isDebugMode && screen != null)
{
Imgproc.rectangle(debugMat, new Point(r.x, r.y), new Point(r.x + r.width, r.y + r.height), new Scalar(255, 0, 0, 255), 2);
}
}
}
//Imgproc.putText (debugMat, "W:" + debugMat.width () + " H:" + debugMat.height () + " SO:" + Screen.orientation, new Point (5, debugMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
if (isDebugMode && screen != null)
{
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(debugMat, debugTexture, debugColors);
}
}
}
// Update is called once per frame
void Update()
{
if (sourceToMatHelper.IsPlaying() && sourceToMatHelper.DidUpdateThisFrame())
{
Mat rgbMat = sourceToMatHelper.GetMat();
// detect faces.
List <Rect> detectResult = new List <Rect>();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbMat, grayMat, Imgproc.COLOR_RGB2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
foreach (Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
}
// face tracking.
List <TrackedRect> trackedRects = new List <TrackedRect>();
rectangleTracker.UpdateTrackedObjects(detectResult);
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict[openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict[openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
opticalFlowFilterDict[openCVRect.id].Process(rgbMat, points, points);
lowPassFilterDict[openCVRect.id].Process(rgbMat, points, points);
}
landmarkPoints.Add(points);
}
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
trackedRects.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face swapping.
if (landmarkPoints.Count >= 2)
{
int ann = 0, bob = 1;
for (int i = 0; i < landmarkPoints.Count - 1; i += 2)
{
ann = i;
bob = i + 1;
faceSwapper.SwapFaces(rgbMat, landmarkPoints[ann], landmarkPoints[bob], 1);
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < trackedRects.Count; i++)
{
Rect openCVRect = trackedRects[i];
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
//Imgproc.putText (rgbMat, "W:" + rgbMat.width () + " H:" + rgbMat.height () + " SO:" + Screen.orientation, new Point (5, rgbMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbMat, texture);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces.
List <Rect> detectResult = new List <Rect>();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
// convert image to greyscale.
Imgproc.cvtColor(rgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
detectResult = faces.toList();
}
// corrects the deviation of a detection result between OpenCV and Dlib.
foreach (Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
}
// face tracking.
rectangleTracker.UpdateTrackedObjects(detectResult);
List <TrackedRect> trackedRects = new List <TrackedRect>();
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict[openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict[openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// create LUT texture.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
faceMaskColorCorrector.CreateLUTTex(openCVRect.id);
}
else if (openCVRect.state == TrackedState.DELETED)
{
faceMaskColorCorrector.DeleteLUTTex(openCVRect.id);
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
for (int i = 0; i < trackedRects.Count; i++)
{
TrackedRect tr = trackedRects[i];
UnityEngine.Rect rect = new UnityEngine.Rect(tr.x, tr.y, tr.width, tr.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
if (tr.state > TrackedState.NEW && tr.state < TrackedState.DELETED)
{
opticalFlowFilterDict[tr.id].Process(rgbaMat, points, points);
lowPassFilterDict[tr.id].Process(rgbaMat, points, points);
}
}
if (extendForehead)
{
AddForeheadPoints(points);
}
landmarkPoints.Add(points);
}
// face masking.
if (faceMaskTexture != null && landmarkPoints.Count >= 1)
{ // Apply face masking between detected faces and a face mask image.
float maskImageWidth = faceMaskTexture.width;
float maskImageHeight = faceMaskTexture.height;
TrackedRect tr;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects[i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, faceMaskTexture);
}
if (tr.state < TrackedState.DELETED)
{
MaskFace(meshOverlay, tr, landmarkPoints[i], faceLandmarkPointsInMask, maskImageWidth, maskImageHeight);
if (enableColorCorrection)
{
CorrectFaceMaskColor(tr.id, faceMaskMat, rgbaMat, faceLandmarkPointsInMask, landmarkPoints[i]);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
else if (landmarkPoints.Count >= 1)
{ // Apply face masking between detected faces.
float maskImageWidth = texture.width;
float maskImageHeight = texture.height;
TrackedRect tr;
for (int i = 0; i < trackedRects.Count; i++)
{
tr = trackedRects[i];
if (tr.state == TrackedState.NEW)
{
meshOverlay.CreateObject(tr.id, texture);
}
if (tr.state < TrackedState.DELETED)
{
MaskFace(meshOverlay, tr, landmarkPoints[i], landmarkPoints[0], maskImageWidth, maskImageHeight);
if (enableColorCorrection)
{
CorrectFaceMaskColor(tr.id, rgbaMat, rgbaMat, landmarkPoints[0], landmarkPoints[i]);
}
}
else if (tr.state == TrackedState.DELETED)
{
meshOverlay.DeleteObject(tr.id);
}
}
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < detectResult.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(detectResult[i].x, detectResult[i].y, detectResult[i].width, detectResult[i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
}
for (int i = 0; i < trackedRects.Count; i++)
{
UnityEngine.Rect rect = new UnityEngine.Rect(trackedRects[i].x, trackedRects[i].y, trackedRects[i].width, trackedRects[i].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 255, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// draw face points.
if (displayDebugFacePoints)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
DrawFaceLandmark(rgbaMat, landmarkPoints[i], new Scalar(0, 255, 0, 255), 2);
}
}
// display face mask image.
if (faceMaskTexture != null && faceMaskMat != null)
{
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, faceRectInMask, new Scalar(255, 0, 0, 255), 2);
}
if (displayDebugFacePoints)
{
DrawFaceLandmark(faceMaskMat, faceLandmarkPointsInMask, new Scalar(0, 255, 0, 255), 2);
}
float scale = (rgbaMat.width() / 4f) / faceMaskMat.width();
float tx = rgbaMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
if (displayFaceRects || displayDebugFacePointsToggle)
{
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
private void Run()
{
meshOverlay = this.GetComponent <TrackedMeshOverlay> ();
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
enableColorCorrectionToggle.isOn = enableColorCorrection;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
if (imgTexture == null)
{
imgTexture = Resources.Load("family") as Texture2D;
}
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
meshOverlay.UpdateOverlayTransform(gameObject.transform);
meshOverlay.Reset();
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
Mat rgbaMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(imgTexture, rgbaMat);
Debug.Log("rgbaMat ToString " + rgbaMat.ToString());
if (faceLandmarkDetector == null)
{
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
}
faceMaskColorCorrector = faceMaskColorCorrector ?? new FaceMaskColorCorrector();
FrontalFaceChecker frontalFaceChecker = new FrontalFaceChecker(width, height);
// detect faces.
List <Rect> detectResult = new List <Rect> ();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
if (cascade == null)
{
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
}
// if (cascade.empty ()) {
// Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
// }
// convert image to greyscale.
Mat gray = new Mat();
Imgproc.cvtColor(rgbaMat, gray, Imgproc.COLOR_RGBA2GRAY);
MatOfRect faces = new MatOfRect();
Imgproc.equalizeHist(gray, gray);
cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size());
//Debug.Log ("faces " + faces.dump ());
detectResult = faces.toList();
// corrects the deviation of a detection result between OpenCV and Dlib.
foreach (Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
gray.Dispose();
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> > ();
foreach (var openCVRect in detectResult)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
Debug.Log("face : " + rect);
//OpenCVForUnityUtils.DrawFaceRect(imgMat, rect, new Scalar(255, 0, 0, 255), 2);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
landmarkPoints.Add(points);
}
// mask faces.
int[] face_nums = new int[landmarkPoints.Count];
for (int i = 0; i < face_nums.Length; i++)
{
face_nums [i] = i;
}
face_nums = face_nums.OrderBy(i => System.Guid.NewGuid()).ToArray();
float imageWidth = meshOverlay.width;
float imageHeight = meshOverlay.height;
float maskImageWidth = imgTexture.width;
float maskImageHeight = imgTexture.height;
TrackedMesh tm;
for (int i = 0; i < face_nums.Length; i++)
{
meshOverlay.CreateObject(i, imgTexture);
tm = meshOverlay.GetObjectById(i);
Vector3[] vertices = tm.meshFilter.mesh.vertices;
if (vertices.Length == landmarkPoints [face_nums [i]].Count)
{
for (int j = 0; j < vertices.Length; j++)
{
vertices [j].x = landmarkPoints [face_nums [i]] [j].x / imageWidth - 0.5f;
vertices [j].y = 0.5f - landmarkPoints [face_nums [i]] [j].y / imageHeight;
}
}
Vector2[] uv = tm.meshFilter.mesh.uv;
if (uv.Length == landmarkPoints [face_nums [0]].Count)
{
for (int jj = 0; jj < uv.Length; jj++)
{
uv [jj].x = landmarkPoints [face_nums [0]] [jj].x / maskImageWidth;
uv [jj].y = (maskImageHeight - landmarkPoints [face_nums [0]] [jj].y) / maskImageHeight;
}
}
meshOverlay.UpdateObject(i, vertices, null, uv);
if (enableColorCorrection)
{
faceMaskColorCorrector.CreateLUTTex(i);
Texture2D LUTTex = faceMaskColorCorrector.UpdateLUTTex(i, rgbaMat, rgbaMat, landmarkPoints [face_nums [0]], landmarkPoints [face_nums [i]]);
tm.sharedMaterial.SetTexture("_LUTTex", LUTTex);
tm.sharedMaterial.SetFloat("_ColorCorrection", 1f);
}
else
{
tm.sharedMaterial.SetFloat("_ColorCorrection", 0f);
}
// filter non frontal faces.
if (filterNonFrontalFaces && frontalFaceChecker.GetFrontalFaceRate(landmarkPoints [i]) < frontalFaceRateLowerLimit)
{
tm.sharedMaterial.SetFloat("_Fade", 1f);
}
else
{
tm.sharedMaterial.SetFloat("_Fade", 0.3f);
}
}
// draw face rects.
if (displayFaceRects)
{
int ann = face_nums [0];
UnityEngine.Rect rect_ann = new UnityEngine.Rect(detectResult [ann].x, detectResult [ann].y, detectResult [ann].width, detectResult [ann].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_ann, new Scalar(255, 255, 0, 255), 2);
int bob = 0;
for (int i = 1; i < face_nums.Length; i++)
{
bob = face_nums [i];
UnityEngine.Rect rect_bob = new UnityEngine.Rect(detectResult [bob].x, detectResult [bob].y, detectResult [bob].width, detectResult [bob].height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_bob, new Scalar(255, 0, 0, 255), 2);
}
}
// draw face points.
if (displayDebugFacePoints)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, landmarkPoints [i], new Scalar(0, 255, 0, 255), 2);
}
}
Texture2D texture = new Texture2D(rgbaMat.cols(), rgbaMat.rows(), TextureFormat.RGBA32, false);
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(rgbaMat, texture);
gameObject.transform.GetComponent <Renderer> ().material.mainTexture = texture;
frontalFaceChecker.Dispose();
rgbaMat.Dispose();
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// detect faces.
List <Rect> detectResult;
DetectFaces(rgbaMat, out detectResult, useDlibFaceDetecter);
// face tracking.
List <TrackedRect> trackedRects = new List <TrackedRect>();
rectangleTracker.UpdateTrackedObjects(detectResult);
rectangleTracker.GetObjects(trackedRects, true);
// create noise filter.
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state == TrackedState.NEW)
{
if (!lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict.Add(openCVRect.id, new LowPassPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
if (!opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict.Add(openCVRect.id, new OFPointsFilter((int)faceLandmarkDetector.GetShapePredictorNumParts()));
}
}
else if (openCVRect.state == TrackedState.DELETED)
{
if (lowPassFilterDict.ContainsKey(openCVRect.id))
{
lowPassFilterDict[openCVRect.id].Dispose();
lowPassFilterDict.Remove(openCVRect.id);
}
if (opticalFlowFilterDict.ContainsKey(openCVRect.id))
{
opticalFlowFilterDict[openCVRect.id].Dispose();
opticalFlowFilterDict.Remove(openCVRect.id);
}
}
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in trackedRects)
{
if (openCVRect.state > TrackedState.NEW_DISPLAYED && openCVRect.state < TrackedState.NEW_HIDED)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
// apply noise filter.
if (enableNoiseFilter)
{
opticalFlowFilterDict[openCVRect.id].Process(rgbaMat, points, points);
lowPassFilterDict[openCVRect.id].Process(rgbaMat, points, points);
}
landmarkPoints.Add(points);
}
}
// filter non frontal faces.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
trackedRects.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// face changing.
if (faceMaskTexture != null && landmarkPoints.Count >= 1)
{ // Apply face changing between detected faces and a mask image.
if (detectedFaceRect.width == 0.0f || detectedFaceRect.height == 0.0f)
{
DetectFaces(faceMaskMat, out detectResult, useDlibFaceDetecter);
if (detectResult.Count >= 1)
{
detectedFaceRect = new UnityEngine.Rect(detectResult[0].x, detectResult[0].y, detectResult[0].width, detectResult[0].height);
}
}
if (detectedFaceRect.width > 0 || detectedFaceRect.height > 0)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, faceMaskMat);
List <Vector2> souseLandmarkPoint = faceLandmarkDetector.DetectLandmark(detectedFaceRect);
faceChanger.SetTargetImage(rgbaMat);
for (int i = 0; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(faceMaskMat, souseLandmarkPoint, landmarkPoints[i], 1);
}
faceChanger.ChangeFace();
if (displayFaceRects)
{
OpenCVForUnityUtils.DrawFaceRect(faceMaskMat, detectedFaceRect, new Scalar(255, 0, 0, 255), 2);
}
}
}
else if (landmarkPoints.Count >= 2)
{ // Apply face changing between detected faces.
faceChanger.SetTargetImage(rgbaMat);
for (int i = 1; i < landmarkPoints.Count; i++)
{
faceChanger.AddFaceChangeData(rgbaMat, landmarkPoints[0], landmarkPoints[i], 1);
}
faceChanger.ChangeFace();
}
// draw face rects.
if (displayFaceRects)
{
for (int i = 0; i < trackedRects.Count; i++)
{
Rect openCVRect = trackedRects[i];
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceAngles (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, " " + frontalFaceChecker.GetFrontalFaceRate (landmarkPoints [i]), new Point (rect.xMin, rect.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
}
// display face mask image.
if (faceMaskMat != null)
{
float scale = (rgbaMat.width() / 4f) / faceMaskMat.width();
float tx = rgbaMat.width() - faceMaskMat.width() * scale;
float ty = 0.0f;
Mat trans = new Mat(2, 3, CvType.CV_32F);//1.0, 0.0, tx, 0.0, 1.0, ty);
trans.put(0, 0, scale);
trans.put(0, 1, 0.0f);
trans.put(0, 2, tx);
trans.put(1, 0, 0.0f);
trans.put(1, 1, scale);
trans.put(1, 2, ty);
Imgproc.warpAffine(faceMaskMat, rgbaMat, trans, rgbaMat.size(), Imgproc.INTER_LINEAR, Core.BORDER_TRANSPARENT, new Scalar(0));
if (displayFaceRects || displayDebugFacePoints)
{
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
}
}
//Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar (255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.UnityUtils.Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
private void Run()
{
displayFaceRectsToggle.isOn = displayFaceRects;
useDlibFaceDetecterToggle.isOn = useDlibFaceDetecter;
filterNonFrontalFacesToggle.isOn = filterNonFrontalFaces;
useSeamlessCloneToggle.isOn = useSeamlessClone;
displayDebugFacePointsToggle.isOn = displayDebugFacePoints;
if (imgTexture == null)
{
imgTexture = Resources.Load("family") as Texture2D;
}
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = 0;
float height = 0;
width = gameObject.transform.localScale.x;
height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
Mat rgbaMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(imgTexture, rgbaMat);
Debug.Log("rgbaMat ToString " + rgbaMat.ToString());
if (faceLandmarkDetector == null)
{
faceLandmarkDetector = new FaceLandmarkDetector(sp_human_face_68_dat_filepath);
}
FrontalFaceChecker frontalFaceChecker = new FrontalFaceChecker(width, height);
// detect faces.
List <Rect> detectResult = new List <Rect>();
if (useDlibFaceDetecter)
{
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> result = faceLandmarkDetector.Detect();
foreach (var unityRect in result)
{
detectResult.Add(new Rect((int)unityRect.x, (int)unityRect.y, (int)unityRect.width, (int)unityRect.height));
}
}
else
{
if (cascade == null)
{
cascade = new CascadeClassifier(haarcascade_frontalface_alt_xml_filepath);
}
//if (cascade.empty())
//{
// Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
//}
// convert image to greyscale.
Mat gray = new Mat();
Imgproc.cvtColor(rgbaMat, gray, Imgproc.COLOR_RGBA2GRAY);
MatOfRect faces = new MatOfRect();
Imgproc.equalizeHist(gray, gray);
cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size());
//Debug.Log ("faces " + faces.dump ());
detectResult = faces.toList();
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
foreach (Rect r in detectResult)
{
r.y += (int)(r.height * 0.1f);
}
gray.Dispose();
}
// detect face landmark points.
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <List <Vector2> > landmarkPoints = new List <List <Vector2> >();
foreach (var openCVRect in detectResult)
{
UnityEngine.Rect rect = new UnityEngine.Rect(openCVRect.x, openCVRect.y, openCVRect.width, openCVRect.height);
Debug.Log("face : " + rect);
//OpenCVForUnityUtils.DrawFaceRect(imgMat, rect, new Scalar(255, 0, 0, 255), 2);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
landmarkPoints.Add(points);
}
// filter non frontal facea.
if (filterNonFrontalFaces)
{
for (int i = 0; i < landmarkPoints.Count; i++)
{
if (frontalFaceChecker.GetFrontalFaceRate(landmarkPoints[i]) < frontalFaceRateLowerLimit)
{
detectResult.RemoveAt(i);
landmarkPoints.RemoveAt(i);
i--;
}
}
}
// swap faces.
int[] face_nums = new int[landmarkPoints.Count];
for (int i = 0; i < face_nums.Length; i++)
{
face_nums[i] = i;
}
face_nums = face_nums.OrderBy(i => System.Guid.NewGuid()).ToArray();
if (landmarkPoints.Count >= 2)
{
DlibFaceSwapper faceSwapper = new DlibFaceSwapper();
faceSwapper.useSeamlessCloneForPasteFaces = useSeamlessClone;
faceSwapper.isShowingDebugFacePoints = displayDebugFacePoints;
int ann = 0, bob = 0;
for (int i = 0; i < face_nums.Length - 1; i += 2)
{
ann = face_nums[i];
bob = face_nums[i + 1];
faceSwapper.SwapFaces(rgbaMat, landmarkPoints[ann], landmarkPoints[bob], 1);
}
faceSwapper.Dispose();
}
// draw face rects.
if (displayFaceRects)
{
int ann = 0, bob = 0;
for (int i = 0; i < face_nums.Length - 1; i += 2)
{
ann = face_nums[i];
bob = face_nums[i + 1];
UnityEngine.Rect rect_ann = new UnityEngine.Rect(detectResult[ann].x, detectResult[ann].y, detectResult[ann].width, detectResult[ann].height);
UnityEngine.Rect rect_bob = new UnityEngine.Rect(detectResult[bob].x, detectResult[bob].y, detectResult[bob].width, detectResult[bob].height);
Scalar color = new Scalar(UnityEngine.Random.Range(0, 256), UnityEngine.Random.Range(0, 256), UnityEngine.Random.Range(0, 256), 255);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_ann, color, 2);
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect_bob, color, 2);
//Imgproc.putText (rgbaMat, "" + i % 2, new Point (rect_ann.xMin, rect_ann.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, color, 2, Imgproc.LINE_AA, false);
//Imgproc.putText (rgbaMat, "" + (i % 2 + 1), new Point (rect_bob.xMin, rect_bob.yMin - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 1.0, color, 2, Imgproc.LINE_AA, false);
}
}
frontalFaceChecker.Dispose();
Texture2D texture = new Texture2D(rgbaMat.cols(), rgbaMat.rows(), TextureFormat.RGBA32, false);
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(rgbaMat, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
rgbaMat.Dispose();
}
/// <summary>
/// Raises the change face mask button click event.
/// </summary>
public void OnChangeFaceMaskButtonClick()
{
RemoveFaceMask();
if (faceMaskDatas.Count == 0)
{
return;
}
FaceMaskData maskData = faceMaskDatas [faceMaskDataIndex];
faceMaskDataIndex = (faceMaskDataIndex < faceMaskDatas.Count - 1) ? faceMaskDataIndex + 1 : 0;
if (maskData == null)
{
Debug.LogError("maskData == null");
return;
}
if (maskData.image == null)
{
Debug.LogError("image == null");
return;
}
if (maskData.landmarkPoints.Count != 68)
{
Debug.LogError("landmarkPoints.Count != 68");
return;
}
faceMaskTexture = maskData.image;
faceMaskMat = new Mat(faceMaskTexture.height, faceMaskTexture.width, CvType.CV_8UC3);
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(faceMaskTexture, faceMaskMat);
if (maskData.isDynamicMode)
{
faceRectInMask = DetectFace(faceMaskMat);
faceLandmarkPointsInMask = DetectFaceLandmarkPoints(faceMaskMat, faceRectInMask);
maskData.faceRect = faceRectInMask;
maskData.landmarkPoints = faceLandmarkPointsInMask;
}
else
{
faceRectInMask = maskData.faceRect;
faceLandmarkPointsInMask = maskData.landmarkPoints;
}
if (faceRectInMask.width == 0 && faceRectInMask.height == 0)
{
RemoveFaceMask();
Debug.LogError("A face could not be detected from the input image.");
}
enableColorCorrectionToggle.isOn = maskData.enableColorCorrection;
/*
* DumpFaceRect (faceRectInMask);
* DumpLandMarkPoints (faceLandmarkPointsInMask);
*/
/*
* if (maskData.name == "Panda") {
* UnityEngine.Rect faceRect;
* List<Vector2> landmarkPoints;
* CreatePandaMaskData (out faceRect, out landmarkPoints);
* SetFaceMaskData (maskData, faceRect, landmarkPoints);
* }else if (maskData.name == "Anime") {
* UnityEngine.Rect faceRect;
* List<Vector2> landmarkPoints;
* CreateAnimeMaskData (out faceRect, out landmarkPoints);
* SetFaceMaskData (maskData, faceRect, landmarkPoints);
* }
*/
}
