/// <summary>
/// Postprocess the specified frame, outs and net.
/// </summary>
/// <param name="frame">Frame.</param>
/// <param name="outs">Outs.</param>
/// <param name="net">Net.</param>
protected virtual void postprocess(Mat frame, List <Mat> outs, Net net)
{
string outLayerType = outBlobTypes[0];
List <int> classIdsList = new List <int>();
List <float> confidencesList = new List <float>();
List <Rect2d> boxesList = new List <Rect2d>();
if (net.getLayer(new DictValue(0)).outputNameToIndex("im_info") != -1)
{
// Faster-RCNN or R-FCN
// Network produces output blob with a shape 1x1xNx7 where N is a number of
// detections and an every detection is a vector of values
// [batchId, classId, confidence, left, top, right, bottom]
if (outs.Count == 1)
{
outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7);
//Debug.Log ("outs[i].ToString() " + outs [0].ToString ());
float[] data = new float[7];
for (int i = 0; i < outs[0].rows(); i++)
{
outs[0].get(i, 0, data);
float confidence = data[2];
if (confidence > confThreshold)
{
int class_id = (int)(data[1]);
float left = data[3] * frame.cols();
float top = data[4] * frame.rows();
float right = data[5] * frame.cols();
float bottom = data[6] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add((int)(class_id) - 0);
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else if (outLayerType == "DetectionOutput")
{
// Network produces output blob with a shape 1x1xNx7 where N is a number of
// detections and an every detection is a vector of values
// [batchId, classId, confidence, left, top, right, bottom]
if (outs.Count == 1)
{
outs[0] = outs[0].reshape(1, (int)outs[0].total() / 7);
//Debug.Log ("outs[i].ToString() " + outs [0].ToString ());
float[] data = new float[7];
for (int i = 0; i < outs[0].rows(); i++)
{
outs[0].get(i, 0, data);
float confidence = data[2];
if (confidence > confThreshold)
{
int class_id = (int)(data[1]);
float left = data[3] * frame.cols();
float top = data[4] * frame.rows();
float right = data[5] * frame.cols();
float bottom = data[6] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add((int)(class_id) - 0);
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else if (outLayerType == "Region")
{
for (int i = 0; i < outs.Count; ++i)
{
// Network produces output blob with a shape NxC where N is a number of
// detected objects and C is a number of classes + 4 where the first 4
// numbers are [center_x, center_y, width, height]
//Debug.Log ("outs[i].ToString() "+outs[i].ToString());
float[] positionData = new float[5];
float[] confidenceData = new float[outs[i].cols() - 5];
for (int p = 0; p < outs[i].rows(); p++)
{
outs[i].get(p, 0, positionData);
outs[i].get(p, 5, confidenceData);
int maxIdx = confidenceData.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V > working.V) ? max : working).I;
float confidence = confidenceData[maxIdx];
if (confidence > confThreshold)
{
float centerX = positionData[0] * frame.cols();
float centerY = positionData[1] * frame.rows();
float width = positionData[2] * frame.cols();
float height = positionData[3] * frame.rows();
float left = centerX - width / 2;
float top = centerY - height / 2;
classIdsList.Add(maxIdx);
confidencesList.Add((float)confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
}
}
else
{
Debug.Log("Unknown output layer type: " + outLayerType);
}
MatOfRect2d boxes = new MatOfRect2d();
boxes.fromList(boxesList);
MatOfFloat confidences = new MatOfFloat();
confidences.fromList(confidencesList);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
//Debug.Log ("indices.dump () "+indices.dump ());
//Debug.Log ("indices.ToString () "+indices.ToString());
for (int i = 0; i < indices.total(); ++i)
{
int idx = (int)indices.get(i, 0)[0];
Rect2d box = boxesList[idx];
drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y,
box.x + box.width, box.y + box.height, frame);
}
indices.Dispose();
boxes.Dispose();
confidences.Dispose();
}
protected override void postprocess(Mat frame, List <Mat> outs, Net net, int backend = Dnn.DNN_BACKEND_OPENCV)
{
List <int> classIdsList = new List <int>();
List <float> confidencesList = new List <float>();
List <Rect2d> boxesList = new List <Rect2d>();
List <Point[]> pointsList = new List <Point[]>();
if (outs.Count == 2)
{
// reshape mat : outs[0]:[1, x, 4] to [x, 4], outs[1]:[1, x, 2] to [x, 2]
Mat boxes_m = outs[0].reshape(1, new int[] { outs[0].size(1), outs[0].size(2) });
Mat scores_m = outs[1].reshape(1, new int[] { outs[1].size(1), outs[1].size(2) });
//Debug.Log("boxes_m: " + boxes_m);
//Debug.Log("scores_m: " + scores_m);
//Debug.Log("priors: " + priors);
convertLocationsToBoxes(boxes_m, priors, 0.1f, 0.2f);
centerFormToCornerForm(boxes_m);
Mat boxes_0_4 = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(0, 4));
float[] boxes_arr = new float[boxes_0_4.rows() * boxes_0_4.cols()];
MatUtils.copyFromMat(boxes_0_4, boxes_arr);
Mat scores_1_2 = new Mat(scores_m, new Range(0, scores_m.rows()), new Range(1, 2));
float[] confidences_arr = new float[scores_1_2.rows()];
MatUtils.copyFromMat(scores_1_2, confidences_arr);
for (int i = 0; i < boxes_m.rows(); i++)
{
float confidence = confidences_arr[i];
if (confidence > confThreshold)
{
int boxes_index = i * 4;
float left = boxes_arr[boxes_index] * frame.cols();
float top = boxes_arr[boxes_index + 1] * frame.rows();
float right = boxes_arr[boxes_index + 2] * frame.cols();
float bottom = boxes_arr[boxes_index + 3] * frame.rows();
float width = right - left + 1f;
float height = bottom - top + 1f;
classIdsList.Add(0);
confidencesList.Add(confidence);
boxesList.Add(new Rect2d(left, top, width, height));
}
}
if (boxes_m.cols() > 4 && boxes_m.cols() % 2 == 0)
{
Mat points = new Mat(boxes_m, new Range(0, boxes_m.rows()), new Range(4, boxes_m.cols()));
float[] points_arr = new float[points.rows() * points.cols()];
MatUtils.copyFromMat(points, points_arr);
for (int i = 0; i < boxes_m.rows(); i++)
{
float confidence = confidences_arr[i];
if (confidence > confThreshold)
{
int points_index = i * points.cols();
Point[] p_arr = new Point[points.cols() / 2];
for (int index = 0; index < points.cols() / 2; index++)
{
float x = points_arr[points_index + index * 2] * frame.cols();
float y = points_arr[points_index + index * 2 + 1] * frame.rows();
p_arr[index] = new Point(x, y);
}
pointsList.Add(p_arr);
}
}
}
}
MatOfRect2d boxes = new MatOfRect2d();
boxes.fromList(boxesList);
MatOfFloat confidences = new MatOfFloat();
confidences.fromList(confidencesList);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
//Debug.Log("indices.dump () " + indices.dump());
//Debug.Log ("indices.ToString () "+indices.ToString());
for (int i = 0; i < indices.total(); ++i)
{
int idx = (int)indices.get(i, 0)[0];
Rect2d box = boxesList[idx];
drawPred(classIdsList[idx], confidencesList[idx], box.x, box.y,
box.x + box.width, box.y + box.height, frame);
if (pointsList.Count > 0)
{
drawPredPoints(pointsList[idx], frame);
}
}
indices.Dispose();
boxes.Dispose();
confidences.Dispose();
}
/// <summary>
/// Get result form all output
/// </summary>
/// <param name="output"></param>
/// <param name="image"></param>
/// <param name="threshold"></param>
/// <param name="nmsThreshold">threshold for nms</param>
/// <param name="nms">Enable Non-maximum suppression or not</param>
private static void GetResult(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold, bool nms = true)
{
//for nms
List <int> classIds = new List <int>();
List <float> confidences = new List <float>();
List <float> probabilities = new List <float>();
List <Rect2d> boxes = new List <Rect2d>();
var w = image.width();
var h = image.height();
/*
* YOLO3 COCO trainval output
* 0 1 : center 2 3 : w/h
* 4 : confidence 5 ~ 84 : class probability
*/
const int prefix = 5; //skip 0~4
foreach (Mat prob in output)
{
for (int i = 0; i < prob.rows(); i++)
{
var confidence = (float)prob.get(i, 4)[0];
if (confidence > threshold)
{
//get classes probability
Core.MinMaxLocResult minAndMax = Core.minMaxLoc(prob.row(i).colRange(prefix, prob.cols()));
int classes = (int)minAndMax.maxLoc.x;
var probability = (float)prob.get(i, classes + prefix)[0];
if (probability > threshold) //more accuracy, you can cancel it
{
//get center and width/height
float centerX = (float)prob.get(i, 0)[0] * w;
float centerY = (float)prob.get(i, 1)[0] * h;
float width = (float)prob.get(i, 2)[0] * w;
float height = (float)prob.get(i, 3)[0] * h;
if (!nms)
{
// draw result (if don't use NMSBoxes)
Draw(image, classes, confidence, probability, centerX, centerY, width, height);
continue;
}
//put data to list for NMSBoxes
classIds.Add(classes);
confidences.Add(confidence);
probabilities.Add(probability);
boxes.Add(new Rect2d(centerX, centerY, width, height));
}
}
}
}
if (!nms)
{
return;
}
//using non-maximum suppression to reduce overlapping low confidence box
MatOfRect2d bboxes = new MatOfRect2d();
MatOfFloat scores = new MatOfFloat();
MatOfInt indices = new MatOfInt();
bboxes.fromList(boxes);
scores.fromList(probabilities);
Dnn.NMSBoxes(bboxes, scores, threshold, nmsThreshold, indices);
int[] indicesA = indices.toArray();
foreach (var i in indicesA)
{
var box = boxes[i];
Draw(image, classIds[i], confidences[i], probabilities[i], box.x, box.y, box.width, box.height);
}
}