private List <DnnDetectedObject> OptimizeDetections(float threshold, float nmsThreshold, bool nms, List <int> classIds, List <float> confidences, List <float> probabilities, List <Rect2d> boxes)
{
int[] indices;
if (!nms)
{
indices = Enumerable.Range(0, boxes.Count).ToArray();
}
else
{
//using non-maximum suppression to reduce overlapping low confidence box
CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out indices);
_logger.LogDebug($"NMSBoxes drop {confidences.Count - indices.Length} overlapping result.");
}
var result = new List <DnnDetectedObject>();
foreach (var i in indices)
{
var box = boxes[i];
var detection = new DnnDetectedObject()
{
Index = classIds[i],
Label = Labels[classIds[i]],
Color = Colors[classIds[i]],
Probability = probabilities[i],
BoundingBox = box
};
result.Add(detection);
}
return(result);
}
public void DetectAllText(string fileName)
{
const int InputWidth = 320;
const int InputHeight = 320;
const float ConfThreshold = 0.5f;
const float NmsThreshold = 0.4f;
// Load network.
using (Net net = CvDnn.ReadNet(Path.GetFullPath(LocalModelPath)))
using (Mat img = new Mat(fileName))
// Prepare input image
using (var blob = CvDnn.BlobFromImage(img, 1.0, new Size(InputWidth, InputHeight), new Scalar(123.68, 116.78, 103.94), true, false))
{
// Forward Pass
// Now that we have prepared the input, we will pass it through the network. There are two outputs of the network.
// One specifies the geometry of the Text-box and the other specifies the confidence score of the detected box.
// These are given by the layers :
// feature_fusion/concat_3
// feature_fusion/Conv_7/Sigmoid
var outputBlobNames = new string[] { "feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3" };
var outputBlobs = outputBlobNames.Select(_ => new Mat()).ToArray();
net.SetInput(blob);
net.Forward(outputBlobs, outputBlobNames);
Mat scores = outputBlobs[0];
Mat geometry = outputBlobs[1];
// Decode predicted bounding boxes (decode the positions of the text boxes along with their orientation)
Decode(scores, geometry, ConfThreshold, out var boxes, out var confidences);
// Apply non-maximum suppression procedure for filtering out the false positives and get the final predictions
CvDnn.NMSBoxes(boxes, confidences, ConfThreshold, NmsThreshold, out var indices);
// Render detections.
Point2f ratio = new Point2f((float)img.Cols / InputWidth, (float)img.Rows / InputHeight);
for (var i = 0; i < indices.Length; ++i)
{
RotatedRect box = boxes[indices[i]];
Point2f[] vertices = box.Points();
for (int j = 0; j < 4; ++j)
{
vertices[j].X *= ratio.X;
vertices[j].Y *= ratio.Y;
}
for (int j = 0; j < 4; ++j)
{
Cv2.Line(img, (int)vertices[j].X, (int)vertices[j].Y, (int)vertices[(j + 1) % 4].X, (int)vertices[(j + 1) % 4].Y, new Scalar(0, 255, 0), 3);
}
}
ShowImagesWhenDebugMode(img);
}
}
private static (Mat, List <string>, List <Point>, List <float>, List <Rect2d>) GetResult(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold)
{
var classNames = new List <string>();
var centers = new List <Point>();
var classIds = new List <int>();
var confidences = new List <float>();
var probabilities = new List <float>();
var boxes = new List <Rect2d>();
var selectedBoxes = new List <Rect2d>();
var selectedConf = new List <float>();
var w = image.Width;
var h = image.Height;
foreach (var pred in output)
{
for (var i = 0; i < pred.Rows; i++)
{
var confidence = pred.At <float>(i, 4);
if (confidence > threshold)
{
Cv2.MinMaxLoc(pred.Row(i).ColRange(5, pred.Cols), out _, out Point classIdPoint);
var probability = pred.At <float>(i, classIdPoint.X + 5);
if (probability > threshold)
{
var centerX = pred.At <float>(i, 0) * w;
var centerY = pred.At <float>(i, 1) * h;
var width = pred.At <float>(i, 2) * w;
var height = pred.At <float>(i, 3) * h;
classIds.Add(classIdPoint.X);
confidences.Add(confidence);
probabilities.Add(probability);
boxes.Add(new Rect2d(centerX, centerY, width, height));
}
}
}
}
CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out int[] indices);
foreach (var i in indices)
{
var box = boxes[i];
selectedBoxes.Add(box);
selectedConf.Add(confidences[i]);
classNames.Add(_labels[classIds[i]]);
centers.Add(new Point(box.X, box.Y));
Draw(image, classIds[i], confidences[i], box.X, box.Y, box.Width, box.Height);
}
return(image, classNames, centers, selectedConf, selectedBoxes);
}
public void NMSBoxes()
{
var bboxes = new Rect[] {
new Rect(10, 10, 20, 20),
new Rect(100, 100, 20, 20),
new Rect(1000, 1000, 20, 20)
};
var scores = new float[] { 1.0f, 0.1f, 0.6f };
float scoreThreshold = 0.5f;
float nmsThreshold = 0.4f;
CvDnn.NMSBoxes(bboxes, scores, scoreThreshold, nmsThreshold, out var indices);
Assert.Equal(2, indices.Length);
Assert.Equal(0, indices[0]);
Assert.Equal(2, indices[1]);
}
public IEnumerable <(float Confidence, double CenterX, double CenterY, double Width, double Height)> Detect(Mat image)
{
var blob = CvDnn.BlobFromImage(image, scalarFactor, configSize, blobFromImageMeanParams, true, false);
net.SetInput(blob);
net.Forward(outputLayers, outputNames);
var items = new List <(float Confidence, double CenterX, double CenterY, double Width, double Height)>();
foreach (var prob in outputLayers)
{
for (var i = 0; i < prob.Rows; i++)
{
var confidence = prob.At <float>(i, 4);
if (confidence > Defaults.ConfidenceThreshold)
{
Cv2.MinMaxLoc(prob.Row(i).ColRange(5, prob.Cols), out _, out Point max);
var type = max.X;
var label = labels[type];
var probability = prob.At <float>(i, type + 5);
if (probability > Defaults.ConfidenceThreshold && label.Equals("person"))
{
var centerX = prob.At <float>(i, 0) * image.Width;
var centerY = prob.At <float>(i, 1) * image.Height;
var width = prob.At <float>(i, 2) * image.Width;
var height = prob.At <float>(i, 3) * image.Height;
items.Add((confidence, centerX, centerY, width, height));
}
}
}
}
//return items;
CvDnn.NMSBoxes(
items.Select(i => new Rect((int)(i.CenterX - (i.Width / 2)), (int)(i.CenterY - (i.Height / 2)), (int)i.Width, (int)i.Height)),
items.Select(i => i.Confidence),
(float)Defaults.ConfidenceThreshold,
(float)Defaults.NonMaximaSupressionThreshold,
out int[] indices);
for (int i = 0; i < indices.Length; i++)
{
yield return(items[indices[i]]);
}
}
public IEnumerable <(float Confidence, double CenterX, double CenterY, double Width, double Height)> Detect(Mat image)
{
var items = new List <(float Confidence, double CenterX, double CenterY, double Width, double Height)>();
foreach (var item in yolo.Detect(image.ToBytes()))
{
if (item.Type.Equals("person"))
{
items.Add(((float)item.Confidence, item.X + item.Width / 2.0, item.Y + item.Height / 2.0, item.Width, item.Height));
}
}
//return items;
CvDnn.NMSBoxes(
items.Select(i => new Rect((int)(i.CenterX - (i.Width / 2)), (int)(i.CenterY - (i.Height / 2)), (int)i.Width, (int)i.Height)),
items.Select(i => i.Confidence),
(float)Defaults.ConfidenceThreshold,
(float)Defaults.NonMaximaSupressionThreshold,
out int[] indices);
for (int i = 0; i < indices.Length; i++)
{
yield return(items[indices[i]]);
}
}
/// <summary>
/// Get result form all output
/// </summary>
/// <param name="output"></param>
/// <param name="image"></param>
/// <param name="threshold"></param>
/// <param name="nmsThreshold">threshold for nms</param>
/// <param name="nms">Enable Non-maximum suppression or not</param>
private static void GetResult(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold, bool nms = true)
{
//for nms
var classIds = new List <int>();
var confidences = new List <float>();
var probabilities = new List <float>();
var boxes = new List <Rect2d>();
var w = image.Width;
var h = image.Height;
/*
* YOLO3 COCO trainval output
* 0 1 : center 2 3 : w/h
* 4 : confidence 5 ~ 84 : class probability
*/
const int prefix = 5; //skip 0~4
foreach (var prob in output)
{
for (var i = 0; i < prob.Rows; i++)
{
var confidence = prob.At <float>(i, 4);
if (confidence > threshold)
{
//get classes probability
Cv2.MinMaxLoc(prob.Row(i).ColRange(prefix, prob.Cols), out _, out Point max);
var classes = max.X;
var probability = prob.At <float>(i, classes + prefix);
if (probability > threshold) //more accuracy, you can cancel it
{
//get center and width/height
var centerX = prob.At <float>(i, 0) * w;
var centerY = prob.At <float>(i, 1) * h;
var width = prob.At <float>(i, 2) * w;
var height = prob.At <float>(i, 3) * h;
if (!nms)
{
// draw result (if don't use NMSBoxes)
Draw(image, classes, confidence, probability, centerX, centerY, width, height);
continue;
}
//put data to list for NMSBoxes
classIds.Add(classes);
confidences.Add(confidence);
probabilities.Add(probability);
boxes.Add(new Rect2d(centerX, centerY, width, height));
}
}
}
}
if (!nms)
{
return;
}
//using non-maximum suppression to reduce overlapping low confidence box
CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out int[] indices);
Console.WriteLine($"NMSBoxes drop {confidences.Count - indices.Length} overlapping result.");
foreach (var i in indices)
{
var box = boxes[i];
Draw(image, classIds[i], confidences[i], probabilities[i], box.X, box.Y, box.Width, box.Height);
}
}
public Rectangle[] Detect(Mat mat, byte[] buffer)
{
var matWidth = mat.Width;
var matHeight = mat.Height;
//using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(104, 117, 123), false, false);
//using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(416, 416), mean: new OpenCvSharp.Scalar(0, 0, 0), swapRB: true, crop: false, ddepth: MatType.CV_8U); // , ddepth: MatType.CV_8U);
//using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(416, 416), mean: new OpenCvSharp.Scalar(0, 0, 0), swapRB: true, crop: false, ddepth: MatType.CV_8U); // , ddepth: MatType.CV_8U);
//Net.SetInput(blob, scaleFactor: 1.0 / 255.0);
using var blob = CvDnn.BlobFromImage(mat, 1.0 / 255.0, new OpenCvSharp.Size(416, 416), crop: false);
Net.SetInput(blob);
var outputLayers = new Mat[OutputLayerNames.Length];
try
{
for (var i = 0; i < outputLayers.Length; i++)
{
outputLayers[i] = new Mat();
}
Net.Forward(outputLayers, OutputLayerNames);
//using var detections = Net.Forward()
//using var detectionMat = new Mat(detections.Size(2), detections.Size(3), MatType.CV_32F, detections.Ptr(0));
var rects = new List <Rect>();
var confidences = new List <float>();
var rectangles = new List <Rectangle>();
for (var i = 0; i < outputLayers.Length; i++)
{
var outs = outputLayers[i];
for (var j = 0; j < outs.Rows; j++)
{
var cols = outs.Row(j).ColRange(5, outs.Cols);
Cv2.MinMaxLoc(cols, out var _, out var confidence, out var _, out var maxVal);
//Cv2.MinMaxLoc(cols, out var minVal, out OpenCvSharp.Point maxVal);
var classes = maxVal.X;
//var confidence = outs.At<float>(j, classes + 5);
if (confidence > 0.5)
{
var centerX = outs.At <float>(j, 0) * matWidth;
var centerY = outs.At <float>(j, 1) * matHeight;
var width = outs.At <float>(j, 2) * matWidth;
var height = outs.At <float>(j, 3) * matHeight;
var left = centerX - (width / 2);
var top = centerY - (height / 2);
rects.Add(new Rect((int)left, (int)top, (int)width, (int)height));
confidences.Add((float)confidence);
Console.WriteLine($"Class {classes} at {centerX},{centerY} of {width}x{height} with confidence {confidence}");
rectangles.Add(new Rectangle((int)(centerX - (width / 2)), (int)(centerY - (height / 2)), (int)(centerX + (width / 2)), (int)(centerY + (height / 2))));
}
}
}
//for (var i = 0; i < detectionMat.Rows; i++)
//{
// var confidence = detectionMat.At<float>(i, 2);
// if (confidence > 0.7)
// {
// var left = (int)(detectionMat.At<float>(i, 3) * frameWidth);
// var top = (int)(detectionMat.At<float>(i, 4) * frameHeight);
// var right = (int)(detectionMat.At<float>(i, 5) * frameWidth);
// var bottom = (int)(detectionMat.At<float>(i, 6) * frameHeight);
// rectangles.Add(new Rectangle(left, top, right - left, bottom - top));
// }
//}
CvDnn.NMSBoxes(rects, confidences, 0.5f, 0.3f, out var indices);
var results = new List <Rectangle>(indices.Length);
foreach (var index in indices)
{
results.Add(rectangles[index]);
}
return(results.ToArray());
}
finally
{
foreach (var outputLayer in outputLayers)
{
outputLayer?.Dispose();
}
}
}
private DnnDetectedObject[] ExtractYolo2Results(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold, bool nms = true)
{
var classIds = new List <int>();
var confidences = new List <float>();
var probabilities = new List <float>();
var boxes = new List <Rect2d>();
var w = image.Width;
var h = image.Height;
var prob = output.First();
/* YOLO2 VOC output
* 0 1 : center 2 3 : w/h
* 4 : confidence 5 ~24 : class probability */
const int prefix = 5; //skip 0~4
for (int i = 0; i < prob.Rows; i++)
{
var confidence = prob.At <float>(i, 4);
if (confidence > threshold)
{
//get classes probability
Cv2.MinMaxLoc(prob.Row(i).ColRange(prefix, prob.Cols), out _, out Point max);
var classes = max.X;
var probability = prob.At <float>(i, classes + prefix);
if (probability > threshold) //more accuracy
{
//get center and width/height
var centerX = prob.At <float>(i, 0) * w;
var centerY = prob.At <float>(i, 1) * h;
var width = prob.At <float>(i, 2) * w;
var height = prob.At <float>(i, 3) * h;
float X = Math.Max(0, centerX - (width / 2.0f));
float Y = Math.Max(0, centerY - (height / 2.0f));
//put data to list for NMSBoxes
classIds.Add(classes);
confidences.Add(confidence);
probabilities.Add(probability);
boxes.Add(new Rect2d(X, Y, width, height));
}
}
}
int[] indices;
if (!nms)
{
//using non-maximum suppression to reduce overlapping low confidence box
indices = Enumerable.Range(0, boxes.Count).ToArray();
}
else
{
CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out indices);
_logger.LogInformation($"NMSBoxes drop {confidences.Count - indices.Length} overlapping result.");
}
var result = new List <DnnDetectedObject>();
foreach (var i in indices)
{
var box = boxes[i];
var detection = new DnnDetectedObject()
{
Index = classIds[i],
Label = Labels[classIds[i]],
Color = Colors[classIds[i]],
Probability = probabilities[i],
BoundingBox = box
};
result.Add(detection);
}
return(result.ToArray());
}
/// <summary>
/// Get result form all output
/// </summary>
/// <param name="outputs"></param>
/// <param name="image"></param>
/// <param name="confidenceThreshold"></param>
/// <param name="nmsThreshold">threshold for nms</param>
/// <param name="nms">Enable Non-maximum suppression or not</param>
private void DrawResults(IEnumerable <Mat> outputs, Mat image, float confidenceThreshold, float nmsThreshold, bool nms = true)
{
//for nms
var classIds = new List <int>();
var confidences = new List <float>();
var probabilities = new List <float>();
var boxes = new List <Rect2d>();
var w = image.Width;
var h = image.Height;
/*
* YOLO3 COCO trainval output
* 0 1 : center
* 2 3 : w/h
* 4 : confidence
* 5 ~ 84 : class probability
*/
const int prefix = 5; //skip 0~4
foreach (var output in outputs)
{
for (var i = 0; i < output.Rows; i++)
{
var confidence = output.At <float>(i, 4);
if (confidence > confidenceThreshold)
{
//get classes probability
Cv2.MinMaxLoc(output.Row(i).ColRange(prefix, output.Cols), out _, out double max);
var classes = (int)max;
var probability = output.At <float>(i, classes + prefix);
if (probability > confidenceThreshold) //more accuracy, you can cancel it
{
//get center and width/height
var centerX = output.At <float>(i, 0) * w;
var centerY = output.At <float>(i, 1) * h;
var width = output.At <float>(i, 2) * w;
var height = output.At <float>(i, 3) * h;
if (!nms)
{
// draw result (if don't use NMSBoxes)
DrawResult(image, classes, confidence, probability, centerX, centerY, width, height);
continue;
}
//put data to list for NMSBoxes
classIds.Add(classes);
confidences.Add(confidence);
probabilities.Add(probability);
boxes.Add(new Rect2d(centerX, centerY, width, height));
}
}
}
}
if (!nms)
{
return;
}
//using non-maximum suppression to reduce overlapping low confidence box
CvDnn.NMSBoxes(boxes, confidences, confidenceThreshold, nmsThreshold, out int[] indices);
foreach (var i in indices)
{
var box = boxes[i];
DrawResult(image, classIds[i], confidences[i], probabilities[i], box.X, box.Y, box.Width, box.Height);
}
}