/// <summary>
/// Postprocess the specified frame, outs and net.
/// </summary>
/// <param name="frame">Frame.</param>
/// <param name="outs">Outs.</param>
/// <param name="net">Net.</param>
protected virtual void postprocess(Mat frame, List <Mat> outs, Net net)
{
string outLayerType = outBlobTypes[0];
List <int> classIdsList = new List <int>();
List <float> confidencesList = new List <float>();
List <Rect2d> boxesList = new List <Rect2d>();
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{
// Faster-RCNN or R-FCN
// Network produces output blob with a shape 1x1xNx7 where N is a number of
// detections and an every detection is a vector of values
// [batchId, classId, confidence, left, top, right, bottom]
if (outs.Count == 1)
{
outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7);
//Debug.Log ("outs[i].ToString() " + outs [0].ToString ());
float[] data = new float[7];
for (int i = 0; i < outs[0].rows(); i++)
{
outs[0].get(i, 0, data);
float confidence = data[2];
if (confidence > confThreshold)
{
int class_id = (int)(data[1]);
float left = data[3] * frame.cols();
float top = data[4] * frame.rows();
float right = data[5] * frame.cols();
float bottom = data[6] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add((int)(class_id) - 0);
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else if (outLayerType == "DetectionOutput")
{
// Network produces output blob with a shape 1x1xNx7 where N is a number of
// detections and an every detection is a vector of values
// [batchId, classId, confidence, left, top, right, bottom]
if (outs.Count == 1)
{
//Debug.Log(outs.Count);
//Debug.Log(outs[0]);
outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7);
//Debug.Log ("outs[i].ToString() " + outs [0].ToString ());
float[] data = new float[7];
for (int i = 0; i < outs[0].rows(); i++)
{
outs[0].get(i, 0, data);
//Debug.Log("data[1] " + data[1]);
float confidence = data[2];
if (confidence > confThreshold)
{
int class_id = (int)(data[1]);
float left = data[3] * frame.cols();
float top = data[4] * frame.rows();
float right = data[5] * frame.cols();
float bottom = data[6] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add((int)(class_id) - 0);
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else if (outLayerType == "Region")
{
for (int i = 0; i < outs.Count; ++i)
{
// Network produces output blob with a shape NxC where N is a number of
// detected objects and C is a number of classes + 4 where the first 4
// numbers are [center_x, center_y, width, height]
//Debug.Log ("outs[i].ToString() "+outs[i].ToString());
float[] positionData = new float[5];
float[] confidenceData = new float[outs[i].cols() - 5];
for (int p = 0; p < outs[i].rows(); p++)
{
outs[i].get(p, 0, positionData);
outs[i].get(p, 5, confidenceData);
int maxIdx = confidenceData.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I;
float confidence = confidenceData[maxIdx];
if (confidence > confThreshold)
{
float centerX = positionData[0] * frame.cols();
float centerY = positionData[1] * frame.rows();
float width = positionData[2] * frame.cols();
float height = positionData[3] * frame.rows();
float left = centerX - width / 2;
float top = centerY - height / 2;
classIdsList.Add(maxIdx);
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else
{
Debug.Log("Unknown output layer type: " + outLayerType);
}
MatOfRect2d boxes = new MatOfRect2d();
boxes.fromList(boxesList);
MatOfFloat confidences = new MatOfFloat();
confidences.fromList(confidencesList);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
//Debug.Log ("indices.dump () "+indices.dump ());
//Debug.Log ("indices.ToString () "+indices.ToString());
for (int i = 0; i < indices.total(); ++i)
{
int idx = (int)indices.get(i, 0)[0];
Rect2d box = boxesList[idx];
drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y,
box.x + box.width, box.y + box.height, frame);
}
indices.Dispose();
boxes.Dispose();
confidences.Dispose();
}
/// <summary>
/// Postprocess the specified frame, outs and net.
/// </summary>
/// <param name="frame">Frame.</param>
/// <param name="outs">Outs.</param>
/// <param name="net">Net.</param>
/// <param name="backend">Backend.</param>
protected virtual void postprocess(Mat frame, List <Mat> outs, Net net, int backend = Dnn.DNN_BACKEND_OPENCV)
{
MatOfInt outLayers = net.getUnconnectedOutLayers();
string outLayerType = outBlobTypes[0];
List <int> classIdsList = new List <int>();
List <float> confidencesList = new List <float>();
List <Rect2d> boxesList = new List <Rect2d>();
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{
// Faster-RCNN or R-FCN
// Network produces output blob with a shape 1x1xNx7 where N is a number of
// detections and an every detection is a vector of values
// [batchId, classId, confidence, left, top, right, bottom]
if (outs.Count == 1)
{
outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7);
//Debug.Log ("outs[i].ToString() " + outs [0].ToString ());
float[] data = new float[7];
for (int i = 0; i < outs[0].rows(); i++)
{
outs[0].get(i, 0, data);
float confidence = data[2];
if (confidence > confThreshold)
{
int class_id = (int)(data[1]);
float left = data[3] * frame.cols();
float top = data[4] * frame.rows();
float right = data[5] * frame.cols();
float bottom = data[6] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add((int)(class_id) - 1); // Skip 0th background class id.
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else if (outLayerType == "DetectionOutput")
{
// Network produces output blob with a shape 1x1xNx7 where N is a number of
// detections and an every detection is a vector of values
// [batchId, classId, confidence, left, top, right, bottom]
if (outs.Count == 1)
{
outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7);
//Debug.Log ("outs[i].ToString() " + outs [0].ToString ());
float[] data = new float[7];
for (int i = 0; i < outs[0].rows(); i++)
{
outs[0].get(i, 0, data);
float confidence = data[2];
if (confidence > confThreshold)
{
int class_id = (int)(data[1]);
float left = data[3] * frame.cols();
float top = data[4] * frame.rows();
float right = data[5] * frame.cols();
float bottom = data[6] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add((int)(class_id) - 1); // Skip 0th background class id.
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else if (outLayerType == "Region")
{
for (int i = 0; i < outs.Count; ++i)
{
// Network produces output blob with a shape NxC where N is a number of
// detected objects and C is a number of classes + 4 where the first 4
// numbers are [center_x, center_y, width, height]
//Debug.Log ("outs[i].ToString() "+outs[i].ToString());
float[] positionData = new float[5];
float[] confidenceData = new float[outs[i].cols() - 5];
for (int p = 0; p < outs[i].rows(); p++)
{
outs[i].get(p, 0, positionData);
outs[i].get(p, 5, confidenceData);
int maxIdx = confidenceData.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I;
float confidence = confidenceData[maxIdx];
if (confidence > confThreshold)
{
float centerX = positionData[0] * frame.cols();
float centerY = positionData[1] * frame.rows();
float width = positionData[2] * frame.cols();
float height = positionData[3] * frame.rows();
float left = centerX - width / 2;
float top = centerY - height / 2;
classIdsList.Add(maxIdx);
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else
{
Debug.Log("Unknown output layer type: " + outLayerType);
}
// NMS is used inside Region layer only on DNN_BACKEND_OPENCV for another backends we need NMS in sample
// or NMS is required if number of outputs > 1
if (outLayers.total() > 1 || (outLayerType == "Region" && backend != Dnn.DNN_BACKEND_OPENCV))
{
Dictionary <int, List <int> > class2indices = new Dictionary <int, List <int> >();
for (int i = 0; i < classIdsList.Count; i++)
{
if (confidencesList[i] >= confThreshold)
{
if (!class2indices.ContainsKey(classIdsList[i]))
{
class2indices.Add(classIdsList[i], new List <int>());
}
class2indices[classIdsList[i]].Add(i);
}
}
List <Rect2d> nmsBoxesList = new List <Rect2d>();
List <float> nmsConfidencesList = new List <float>();
List <int> nmsClassIdsList = new List <int>();
foreach (int key in class2indices.Keys)
{
List <Rect2d> localBoxesList = new List <Rect2d>();
List <float> localConfidencesList = new List <float>();
List <int> classIndicesList = class2indices[key];
for (int i = 0; i < classIndicesList.Count; i++)
{
localBoxesList.Add(boxesList[classIndicesList[i]]);
localConfidencesList.Add(confidencesList[classIndicesList[i]]);
}
using (MatOfRect2d localBoxes = new MatOfRect2d(localBoxesList.ToArray()))
using (MatOfFloat localConfidences = new MatOfFloat(localConfidencesList.ToArray()))
using (MatOfInt nmsIndices = new MatOfInt())
{
Dnn.NMSBoxes(localBoxes, localConfidences, confThreshold, nmsThreshold, nmsIndices);
for (int i = 0; i < nmsIndices.total(); i++)
{
int idx = (int)nmsIndices.get(i, 0)[0];
nmsBoxesList.Add(localBoxesList[idx]);
nmsConfidencesList.Add(localConfidencesList[idx]);
nmsClassIdsList.Add(key);
}
}
}
boxesList = nmsBoxesList;
classIdsList = nmsClassIdsList;
confidencesList = nmsConfidencesList;
}
for (int idx = 0; idx < boxesList.Count; ++idx)
{
Rect2d box = boxesList[idx];
drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y,
box.x + box.width, box.y + box.height, frame);
}
}
// Update is called once per frame
void Update()
{
for (int i = 0; i < validTimer.Length; i++)
{
validTimer[i] -= Time.deltaTime;
if (validTimer[i] <= 0.0f)
{
timerEnded(i);
}
}
switch (ClassID)
{
case 20: //LCD
GUIObject[3].transform.position = Vector3.MoveTowards(GUIObject[3].transform.position, sphere[3].transform.position, Time.deltaTime * speed);
break;
case 12: //Dog
GUIObject[3].transform.position = Vector3.MoveTowards(GUIObject[3].transform.position, sphere[3].transform.position, Time.deltaTime * speed);
break;
case 9: // Chair
GUIObject[2].transform.position = Vector3.MoveTowards(GUIObject[2].transform.position, sphere[2].transform.position, Time.deltaTime * speed);
break;
case 5: // Bottle
GUIObject[1].transform.position = Vector3.MoveTowards(GUIObject[1].transform.position, sphere[1].transform.position, Time.deltaTime * speed);
break;
case 15: // Civilian
GUIObject[0].transform.position = Vector3.MoveTowards(GUIObject[0].transform.position, sphere[0].transform.position, Time.deltaTime * speed);
break;
default:
print("Unknown");
break;
}
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : bgrMat.cols(),
inpHeight > 0 ? inpHeight : bgrMat.rows());
Mat blob = Dnn.blobFromImage(bgrMat, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1) // Faster-RCNN or R-FCN
{
Imgproc.resize(bgrMat, bgrMat, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat> ();
net.forward(outs, outBlobNames);
tm.stop();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
postprocess(rgbaMat, outs, net);
for (int i = 0; i < outs.Count; i++)
{
outs [i].Dispose();
}
blob.Dispose();
}
Utils.fastMatToTexture2D(rgbaMat, texture);
}
}
// Use this for initialization
protected virtual void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
if (!string.IsNullOrEmpty(classes))
{
classNames = readClassNames(classes_filepath);
if (classNames == null)
{
Debug.LogError(classes_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
}
else if (classesList.Count > 0)
{
classNames = classesList;
}
Mat img = Imgcodecs.imread(input_filepath);
if (img.empty())
{
Debug.LogError(input_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(424, 640, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net net = null;
if (string.IsNullOrEmpty(model_filepath))
{
Debug.LogError(model_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
//! [Initialize network]
net = Dnn.readNet(model_filepath, config_filepath);
//! [Initialize network]
}
if (net == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
outBlobNames = getOutputsNames(net);
//for (int i = 0; i < outBlobNames.Count; i++)
//{
// Debug.Log("names [" + i + "] " + outBlobNames[i]);
//}
outBlobTypes = getOutputsTypes(net);
//for (int i = 0; i < outBlobTypes.Count; i++)
//{
// Debug.Log("types [" + i + "] " + outBlobTypes[i]);
//}
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : img.cols(),
inpHeight > 0 ? inpHeight : img.rows());
Mat blob = Dnn.blobFromImage(img, scale, inpSize, mean, swapRB, false);
// Run a model.
net.setInput(blob);
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{ // Faster-RCNN or R-FCN
Imgproc.resize(img, img, inpSize);
Mat imInfo = new Mat(1, 3, CvType.CV_32FC1);
imInfo.put(0, 0, new float[] {
(float)inpSize.height,
(float)inpSize.width,
1.6f
});
net.setInput(imInfo, "im_info");
}
TickMeter tm = new TickMeter();
tm.start();
List <Mat> outs = new List <Mat>();
net.forward(outs, outBlobNames);
tm.stop();
Debug.Log("Inference time, ms: " + tm.getTimeMilli());
postprocess(img, outs, net, Dnn.DNN_BACKEND_OPENCV);
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
net.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Use this for initialization
void Run()
{
//if true, The error log of the Native side OpenCV will be displayed on the Unity Editor Console.
Utils.setDebugMode(true);
Mat img = Imgcodecs.imread(image_filepath, Imgcodecs.IMREAD_COLOR);
if (img.empty())
{
Debug.LogError(image_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
img = new Mat(368, 368, CvType.CV_8UC3, new Scalar(0, 0, 0));
}
//Adust Quad.transform.localScale.
gameObject.transform.localScale = new Vector3(img.width(), img.height(), 1);
Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
float imageWidth = img.width();
float imageHeight = img.height();
float widthScale = (float)Screen.width / imageWidth;
float heightScale = (float)Screen.height / imageHeight;
if (widthScale < heightScale)
{
Camera.main.orthographicSize = (imageWidth * (float)Screen.height / (float)Screen.width) / 2;
}
else
{
Camera.main.orthographicSize = imageHeight / 2;
}
Net detector = null;
Net recognizer = null;
if (string.IsNullOrEmpty(detectionmodel_filepath) || string.IsNullOrEmpty(recognitionmodel_filepath))
{
Debug.LogError(detectionmodel_filepath + " or " + recognitionmodel_filepath + " is not loaded. Please see \"StreamingAssets/dnn/setup_dnn_module.pdf\". ");
}
else
{
detector = Dnn.readNet(detectionmodel_filepath);
recognizer = Dnn.readNet(recognitionmodel_filepath);
}
if (detector == null || recognizer == null)
{
Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
TickMeter tickMeter = new TickMeter();
List <Mat> outs = new List <Mat>();
List <string> outNames = new List <string>();
outNames.Add("feature_fusion/Conv_7/Sigmoid");
outNames.Add("feature_fusion/concat_3");
// Create a 4D blob from a frame.
Size inpSize = new Size(inpWidth > 0 ? inpWidth : img.cols(), inpHeight > 0 ? inpHeight : img.rows());
Mat blob = Dnn.blobFromImage(img, 1.0, inpSize, new Scalar(123.68, 116.78, 103.94), true, false); // blobFromImage(frame, blob, 1.0, Size(inpWidth, inpHeight), Scalar(123.68, 116.78, 103.94), true, false);
// Run detection model.
detector.setInput(blob);
tickMeter.start();
detector.forward(outs, outNames);
tickMeter.stop();
Mat scores = outs[0];
Mat geometry = outs[1];
// Decode predicted bounding boxes.
List <RotatedRect> boxes = new List <RotatedRect>();
List <float> confidences = new List <float>();
decodeBoundingBoxes(scores, geometry, confThreshold, boxes, confidences);
// Apply non-maximum suppression procedure.
MatOfRotatedRect boxesMat = new MatOfRotatedRect(boxes.ToArray());
MatOfFloat confidencesMat = new MatOfFloat(confidences.ToArray());
MatOfInt indicesMat = new MatOfInt();
Dnn.NMSBoxesRotated(boxesMat, confidencesMat, confThreshold, nmsThreshold, indicesMat);
List <int> indices = indicesMat.toList();
Point ratio = new Point(img.cols() / inpWidth, img.rows() / inpHeight);
// Render text.
for (int i = 0; i < indices.Count; ++i)
{
RotatedRect box = boxes[indices[i]];
Point[] vertices = new Point[4];
box.points(vertices);
for (int j = 0; j < 4; ++j)
{
vertices[j].x *= ratio.x;
vertices[j].y *= ratio.y;
}
for (int j = 0; j < 4; ++j)
{
Imgproc.line(img, vertices[j], vertices[(j + 1) % 4], new Scalar(0, 255, 0), 1);
}
if (recognizer != null)
{
Mat cropped = new Mat();
fourPointsTransform(img, vertices, cropped);
//Debug.Log(cropped);
Imgproc.cvtColor(cropped, cropped, Imgproc.COLOR_BGR2GRAY);
Mat blobCrop = Dnn.blobFromImage(cropped, 1.0 / 127.5, new Size(), Scalar.all(127.5));
recognizer.setInput(blobCrop);
//Debug.Log(blobCrop);
tickMeter.start();
Mat result = recognizer.forward();
tickMeter.stop();
string wordRecognized;
decodeText(result, out wordRecognized);
Imgproc.putText(img, wordRecognized, vertices[1], Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 0, 0), 1, Imgproc.LINE_AA, false);
Debug.Log(wordRecognized);
cropped.Dispose();
blobCrop.Dispose();
result.Dispose();
}
}
Debug.Log("Inference time, ms: " + tickMeter.getTimeMilli());
for (int i = 0; i < outs.Count; i++)
{
outs[i].Dispose();
}
blob.Dispose();
detector.Dispose();
recognizer.Dispose();
}
Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
Texture2D texture = new Texture2D(img.cols(), img.rows(), TextureFormat.RGBA32, false);
Utils.matToTexture2D(img, texture);
gameObject.GetComponent <Renderer>().material.mainTexture = texture;
Utils.setDebugMode(false);
}
// Update is called once per frame
void Update()
{
if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
{
Mat rgbaMat = webCamTextureToMatHelper.GetMat();
if (net == null)
{
Imgproc.putText(rgbaMat, "model file is not loaded.", new Point(5, rgbaMat.rows() - 30), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
Imgproc.putText(rgbaMat, "Please read console message.", new Point(5, rgbaMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
}
else
{
Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR);
//! [Resizing without keeping aspect ratio]
Imgproc.resize(bgrMat, resized, new Size(network_width, network_height));
//! [Resizing without keeping aspect ratio]
//! [Prepare blob]
inputBlob = Dnn.blobFromImage(resized, 1 / 255.0, new Size(), new Scalar(0), true, true); //Convert Mat to batch of images
//! [Prepare blob]
//! [Set input blob]
net.setInput(inputBlob, "data"); //set the network input
//! [Set input blob]
// TickMeter tm = new TickMeter ();
// tm.start ();
//! [Make forward pass]
Mat detectionMat = net.forward("detection_out"); //compute output
//! [Make forward pass]
// tm.stop ();
// Debug.Log ("Inference time, ms: " + tm.getTimeMilli ());
// Debug.Log ("detectionMat.ToString(): " + detectionMat.ToString ());
float[] position = new float[5];
float[] confidences = new float[80];
float confidenceThreshold = 0.24f;
for (int i = 0; i < detectionMat.rows(); i++)
{
detectionMat.get(i, 0, position);
detectionMat.get(i, 5, confidences);
int maxIdx = confidences.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I;
float confidence = confidences [maxIdx];
if (confidence > confidenceThreshold)
{
float x = position [0];
float y = position [1];
float width = position [2];
float height = position [3];
int xLeftBottom = (int)((x - width / 2) * rgbaMat.cols());
int yLeftBottom = (int)((y - height / 2) * rgbaMat.rows());
int xRightTop = (int)((x + width / 2) * rgbaMat.cols());
int yRightTop = (int)((y + height / 2) * rgbaMat.rows());
// Debug.Log ("confidence: " + confidence);
//
// Debug.Log (" " + xLeftBottom
// + " " + yLeftBottom
// + " " + xRightTop
// + " " + yRightTop);
Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom), new Point(xRightTop, yRightTop),
new Scalar(0, 255, 0, 255), 2);
if (maxIdx < classNames.Count)
{
string label = classNames [maxIdx] + ": " + confidence;
int[] baseLine = new int[1];
Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine);
Imgproc.rectangle(rgbaMat, new Point(xLeftBottom, yLeftBottom),
new Point(xLeftBottom + labelSize.width, yLeftBottom + labelSize.height + baseLine [0]),
new Scalar(255, 255, 255, 255), Core.FILLED);
Imgproc.putText(rgbaMat, label, new Point(xLeftBottom, yLeftBottom + labelSize.height),
Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0, 255));
}
}
}
detectionMat.Dispose();
}
Utils.matToTexture2D(rgbaMat, texture, webCamTextureToMatHelper.GetBufferColors());
}
}
