// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.isPlaying() && webCamTextureToMatHelper.didUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
if (points.Count > 0)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0, 255), 2);
}
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors);
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.isPlaying() && webCamTextureToMatHelper.didUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
// Resize image for face detection
Imgproc.resize(rgbaMat, rgbaMat_downscale, new Size(), 1.0 / FACE_DOWNSAMPLE_RATIO, 1.0 / FACE_DOWNSAMPLE_RATIO, Imgproc.INTER_LINEAR);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat_downscale);
// Detect faces on resize image
if (count % SKIP_FRAMES == 0)
{
//detect face rects
detectResult = faceLandmarkDetector.Detect();
}
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
if (points.Count > 0)
{
List <Vector2> originalPoints = new List <Vector2> (points.Count);
foreach (var point in points)
{
originalPoints.Add(new Vector2(point.x * FACE_DOWNSAMPLE_RATIO, point.y * FACE_DOWNSAMPLE_RATIO));
}
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, originalPoints, new Scalar(0, 255, 0, 255), 2);
}
UnityEngine.Rect originalRect = new UnityEngine.Rect(rect.x * FACE_DOWNSAMPLE_RATIO, rect.y * FACE_DOWNSAMPLE_RATIO, rect.width * FACE_DOWNSAMPLE_RATIO, rect.height * FACE_DOWNSAMPLE_RATIO);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, originalRect, new Scalar(255, 0, 0, 255), 2);
}
Imgproc.putText(rgbaMat, "Original: (" + rgbaMat.width() + "," + rgbaMat.height() + ") DownScale; (" + rgbaMat_downscale.width() + "," + rgbaMat_downscale.height() + ") SkipFrames: " + SKIP_FRAMES, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors);
count++;
}
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.isPlaying() && webCamTextureToMatHelper.didUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0, 255), 2);
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
if (points.Count > 0)
{
live2DModelUpdate(points);
currentFacePoints = points;
break;
}
}
if (isHideCameraImage)
{
Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 255), -1);
}
if (currentFacePoints != null)
{
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, currentFacePoints, new Scalar(0, 255, 0, 255), 2);
}
Imgproc.putText(rgbaMat, "W:" + rgbaMat.width() + " H:" + rgbaMat.height() + " SO:" + Screen.orientation, new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbaMat, texture, colors);
}
}
// Update is called once per frame
void Update()
{
//Loop play
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
//error PlayerLoop called recursively! on iOS.reccomend WebCamTexture.
if (capture.grab())
{
capture.retrieve(rgbMat, 0);
Imgproc.cvtColor(rgbMat, rgbMat, Imgproc.COLOR_BGR2RGB);
//Debug.Log ("Mat toString " + rgbMat.ToString ());
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbMat);
//detect face rects
List <UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
if (points.Count > 0)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbMat, points, new Scalar(0, 255, 0), 2);
}
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbMat, rect, new Scalar(255, 0, 0), 2);
}
Imgproc.putText(rgbMat, "W:" + rgbMat.width() + " H:" + rgbMat.height() + " SO:" + Screen.orientation, new Point(5, rgbMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255), 1, Imgproc.LINE_AA, false);
OpenCVForUnity.Utils.matToTexture2D(rgbMat, texture);
}
}
// Use this for initialization
void Run(Mat imgMat)
{
Utils.setDebugMode(true);
if (string.IsNullOrEmpty(dlibShapePredictorFilePath))
{
Debug.LogError("shape predictor file does not exist. Please copy from “DlibFaceLandmarkDetector/StreamingAssets/” to “Assets/StreamingAssets/” folder. ");
}
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, imgMat);
//detect face rectdetecton
List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();
foreach (var result in detectResult)
{
Debug.Log("rect : " + result.rect);
Debug.Log("detection_confidence : " + result.detection_confidence);
Debug.Log("weight_index : " + result.weight_index);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(result.rect);
Debug.Log("face points count : " + points.Count);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2, true);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(imgMat, result, new Scalar(255, 0, 0, 255), 2);
}
faceLandmarkDetector.Dispose();
Utils.setDebugMode(false);
}
///// <summary>
///// Modify `data` by applying a posterization transformation to it.
///// </summary>
///// <param name="data">The byte array to modify.</param>
///// <param name="levels">The threshold levels to split each byte into.</param>
//public static void ProcessImage(byte[] data, byte levels)
//{
// byte factor = (byte) (byte.MaxValue / levels);
// for (int i = 0; i < data.Length; i++)
// {
// data[i] = (byte) (data[i] / factor * factor);
// }
//}
private void ProcessMat(Mat frameMat)
{
if (grayMat == null)
{
grayMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC1);
}
// detect faces on the downscale image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
Mat downScaleRgbaMat = null;
float DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(frameMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = frameMat;
DOWNSCALE_RATIO = 1.0f;
}
// set the downscale mat
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, downScaleRgbaMat);
//detect face rects
if (useOpenCVFaceDetector)
{
// convert image to greyscale.
Imgproc.cvtColor(downScaleRgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
List <OpenCVForUnity.CoreModule.Rect> opencvDetectResult = faces.toList();
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
detectionResult.Clear();
foreach (var opencvRect in opencvDetectResult)
{
detectionResult.Add(new UnityEngine.Rect((float)opencvRect.x, (float)opencvRect.y + (float)(opencvRect.height * 0.1f), (float)opencvRect.width, (float)opencvRect.height));
}
}
}
else
{
// Dlib's face detection processing time increases in proportion to image size.
detectionResult = faceLandmarkDetector.Detect();
}
if (enableDownScale)
{
for (int i = 0; i < detectionResult.Count; ++i)
{
var rect = detectionResult[i];
detectionResult[i] = new UnityEngine.Rect(
rect.x * DOWNSCALE_RATIO,
rect.y * DOWNSCALE_RATIO,
rect.width * DOWNSCALE_RATIO,
rect.height * DOWNSCALE_RATIO);
}
}
}
// set the original scale image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, frameMat);
// detect face landmarks on the original image
foreach (var rect in detectionResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(frameMat, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(frameMat, rect, new Scalar(255, 0, 0, 255), 2);
}
}
// Use this for initialization
void Start()
{
gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float width = gameObject.transform.localScale.x;
float height = gameObject.transform.localScale.y;
float widthScale = (float)Screen.width / width;
float heightScale = (float)Screen.height / height;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = height / 2;
}
Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4);
OpenCVForUnity.Utils.texture2DToMat(imgTexture, imgMat);
Debug.Log("imgMat dst ToString " + imgMat.ToString());
FaceLandmarkDetector faceLandmarkDetector = new FaceLandmarkDetector(DlibFaceLandmarkDetector.Utils.getFilePath("shape_predictor_68_face_landmarks.dat"));
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, imgMat);
//detect face rectdetecton
List <FaceLandmarkDetector.RectDetection> detectResult = faceLandmarkDetector.DetectRectDetection();
foreach (var result in detectResult)
{
Debug.Log("rect : " + result.rect);
Debug.Log("detection_confidence : " + result.detection_confidence);
Debug.Log("weight_index : " + result.weight_index);
// Debug.Log ("face : " + rect);
Imgproc.putText(imgMat, "" + result.detection_confidence, new Point(result.rect.xMin, result.rect.yMin - 20), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
Imgproc.putText(imgMat, "" + result.weight_index, new Point(result.rect.xMin, result.rect.yMin - 5), Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(result.rect);
Debug.Log("face points count : " + points.Count);
if (points.Count > 0)
{
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(imgMat, points, new Scalar(0, 255, 0, 255), 2);
}
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(imgMat, result.rect, new Scalar(255, 0, 0, 255), 2);
}
faceLandmarkDetector.Dispose();
Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false);
OpenCVForUnity.Utils.matToTexture2D(imgMat, texture);
gameObject.GetComponent <Renderer> ().material.mainTexture = texture;
}