public override void UpdateValue()
{
if (capture == null)
{
return;
}
didUpdateResultMat = false;
if (shouldUpdateVideoFrame)
{
shouldUpdateVideoFrame = false;
//Loop play
if (capture.get(Videoio.CAP_PROP_POS_FRAMES) >= capture.get(Videoio.CAP_PROP_FRAME_COUNT))
{
capture.set(Videoio.CAP_PROP_POS_FRAMES, 0);
}
if (capture.grab() && !imageOptimizationHelper.IsCurrentFrameSkipped())
{
capture.retrieve(captureMat, 0);
Imgproc.cvtColor(captureMat, resultMat, Imgproc.COLOR_BGR2RGBA);
downScaleResultMat = imageOptimizationHelper.GetDownScaleMat(resultMat);
didUpdateResultMat = true;
}
}
}
public override void UpdateValue()
{
didUpdateResultMat = false;
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame() && !imageOptimizationHelper.IsCurrentFrameSkipped())
{
resultMat = webCamTextureToMatHelper.GetMat();
downScaleResultMat = imageOptimizationHelper.GetDownScaleMat(resultMat);
didUpdateResultMat = true;
}
}
/// <summary>
/// Process
/// </summary>
/// <returns></returns>
private async void Process()
{
float DOWNSCALE_RATIO = 1.0f;
while (true)
{
// Check TaskCancel
if (tokenSource.Token.IsCancellationRequested)
{
break;
}
rgbaMat = webCamTextureToMatHelper.GetMat();
// Debug.Log ("rgbaMat.ToString() " + rgbaMat.ToString ());
Mat downScaleRgbaMat = null;
DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(rgbaMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = rgbaMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.cvtColor(downScaleRgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
await Task.Run(() =>
{
// detect faces on the downscale image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : bgrMat.cols(),
inpHeight > 0 ? inpHeight : bgrMat.rows());
Mat blob = Dnn.blobFromImage(bgrMat, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(bgrMat, bgrMat, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
postprocess(bgrMat, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
if (enableDownScale)
{
for (int i = 0; i < _boxesList.Count; ++i)
{
var rect = _boxesList[i];
_boxesList[i] = new OpenCVForUnity.CoreModule.Rect(
(int)(rect.x * DOWNSCALE_RATIO),
(int)(rect.y * DOWNSCALE_RATIO),
(int)(rect.width * DOWNSCALE_RATIO),
(int)(rect.height * DOWNSCALE_RATIO));
}
}
}
//Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
MatOfRect boxes = new MatOfRect();
boxes.fromList(_boxesList);
MatOfFloat confidences = new MatOfFloat();
confidences.fromList(_confidencesList);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
// Debug.Log ("indices.dump () "+indices.dump ());
// Debug.Log ("indices.ToString () "+indices.ToString());
for (int i = 0; i < indices.total(); ++i)
{
int idx = (int)indices.get(i, 0)[0];
OpenCVForUnity.CoreModule.Rect box = _boxesList[idx];
drawPred(_classIdsList[idx], _confidencesList[idx], box.x, box.y,
box.x + box.width, box.y + box.height, rgbaMat);
}
indices.Dispose();
boxes.Dispose();
confidences.Dispose();
}
});
Utils.fastMatToTexture2D(rgbaMat, texture);
Thread.Sleep(10);
}
}
///// <summary>
///// Modify `data` by applying a posterization transformation to it.
///// </summary>
///// <param name="data">The byte array to modify.</param>
///// <param name="levels">The threshold levels to split each byte into.</param>
//public static void ProcessImage(byte[] data, byte levels)
//{
// byte factor = (byte) (byte.MaxValue / levels);
// for (int i = 0; i < data.Length; i++)
// {
// data[i] = (byte) (data[i] / factor * factor);
// }
//}
private void ProcessMat(Mat frameMat)
{
if (grayMat == null)
{
grayMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC1);
}
// detect faces on the downscale image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
Mat downScaleRgbaMat = null;
float DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(frameMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = frameMat;
DOWNSCALE_RATIO = 1.0f;
}
// set the downscale mat
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, downScaleRgbaMat);
//detect face rects
if (useOpenCVFaceDetector)
{
// convert image to greyscale.
Imgproc.cvtColor(downScaleRgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
List <OpenCVForUnity.CoreModule.Rect> opencvDetectResult = faces.toList();
// correct the deviation of the detection result of the face rectangle of OpenCV and Dlib.
detectionResult.Clear();
foreach (var opencvRect in opencvDetectResult)
{
detectionResult.Add(new UnityEngine.Rect((float)opencvRect.x, (float)opencvRect.y + (float)(opencvRect.height * 0.1f), (float)opencvRect.width, (float)opencvRect.height));
}
}
}
else
{
// Dlib's face detection processing time increases in proportion to image size.
detectionResult = faceLandmarkDetector.Detect();
}
if (enableDownScale)
{
for (int i = 0; i < detectionResult.Count; ++i)
{
var rect = detectionResult[i];
detectionResult[i] = new UnityEngine.Rect(
rect.x * DOWNSCALE_RATIO,
rect.y * DOWNSCALE_RATIO,
rect.width * DOWNSCALE_RATIO,
rect.height * DOWNSCALE_RATIO);
}
}
}
// set the original scale image
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, frameMat);
// detect face landmarks on the original image
foreach (var rect in detectionResult)
{
//detect landmark points
List <Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(frameMat, points, new Scalar(0, 255, 0, 255), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(frameMat, rect, new Scalar(255, 0, 0, 255), 2);
}
}
/// <summary>
/// Precess
/// </summary>
/// <returns></returns>
private async void Process()
{
float DOWNSCALE_RATIO = 1.0f;
while (true)
{
// Check TaskCancel
if (tokenSource.Token.IsCancellationRequested)
{
break;
}
rgbaMat = webCamTextureToMatHelper.GetMat();
// Debug.Log ("rgbaMat.ToString() " + rgbaMat.ToString ());
Mat downScaleRgbaMat = null;
DOWNSCALE_RATIO = 1.0f;
if (enableDownScale)
{
downScaleRgbaMat = imageOptimizationHelper.GetDownScaleMat(rgbaMat);
DOWNSCALE_RATIO = imageOptimizationHelper.downscaleRatio;
}
else
{
downScaleRgbaMat = rgbaMat;
DOWNSCALE_RATIO = 1.0f;
}
Imgproc.cvtColor(downScaleRgbaMat, grayMat, Imgproc.COLOR_RGBA2GRAY);
await Task.Run(() =>
{
// detect faces on the downscale image
if (!enableSkipFrame || !imageOptimizationHelper.IsCurrentFrameSkipped())
{
using (Mat equalizeHistMat = new Mat())
using (MatOfRect faces = new MatOfRect())
{
Imgproc.equalizeHist(grayMat, equalizeHistMat);
cascade.detectMultiScale(equalizeHistMat, faces, 1.1f, 2, 0 | Objdetect.CASCADE_SCALE_IMAGE, new Size(equalizeHistMat.cols() * 0.15, equalizeHistMat.cols() * 0.15), new Size());
//Debug.Log("faces.dump() " + faces.dump());
detectResult = faces.toArray();
}
if (enableDownScale)
{
for (int i = 0; i < detectResult.Length; ++i)
{
var rect = detectResult[i];
detectResult[i] = new OpenCVForUnity.CoreModule.Rect(
(int)(rect.x *DOWNSCALE_RATIO),
(int)(rect.y *DOWNSCALE_RATIO),
(int)(rect.width *DOWNSCALE_RATIO),
(int)(rect.height *DOWNSCALE_RATIO));
}
}
//Imgproc.rectangle(rgbaMat, new Point(0, 0), new Point(rgbaMat.width(), rgbaMat.height()), new Scalar(0, 0, 0, 0), -1);
for (int i = 0; i < detectResult.Length; i++)
{
Imgproc.rectangle(rgbaMat, new Point(detectResult[i].x, detectResult[i].y), new Point(detectResult[i].x + detectResult[i].width, detectResult[i].y + detectResult[i].height), new Scalar(255, 0, 0, 255), 4);
}
}
});
Utils.fastMatToTexture2D(rgbaMat, texture);
Thread.Sleep(10);
}
}
