private List <DnnDetectedObject> OptimizeDetections(float threshold, float nmsThreshold, bool nms, List <int> classIds, List <float> confidences, List <float> probabilities, List <Rect2d> boxes) { int[] indices; if (!nms) { indices = Enumerable.Range(0, boxes.Count).ToArray(); } else { //using non-maximum suppression to reduce overlapping low confidence box CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out indices); _logger.LogDebug($"NMSBoxes drop {confidences.Count - indices.Length} overlapping result."); } var result = new List <DnnDetectedObject>(); foreach (var i in indices) { var box = boxes[i]; var detection = new DnnDetectedObject() { Index = classIds[i], Label = Labels[classIds[i]], Color = Colors[classIds[i]], Probability = probabilities[i], BoundingBox = box }; result.Add(detection); } return(result); }
public void DetectAllText(string fileName) { const int InputWidth = 320; const int InputHeight = 320; const float ConfThreshold = 0.5f; const float NmsThreshold = 0.4f; // Load network. using (Net net = CvDnn.ReadNet(Path.GetFullPath(LocalModelPath))) using (Mat img = new Mat(fileName)) // Prepare input image using (var blob = CvDnn.BlobFromImage(img, 1.0, new Size(InputWidth, InputHeight), new Scalar(123.68, 116.78, 103.94), true, false)) { // Forward Pass // Now that we have prepared the input, we will pass it through the network. There are two outputs of the network. // One specifies the geometry of the Text-box and the other specifies the confidence score of the detected box. // These are given by the layers : // feature_fusion/concat_3 // feature_fusion/Conv_7/Sigmoid var outputBlobNames = new string[] { "feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3" }; var outputBlobs = outputBlobNames.Select(_ => new Mat()).ToArray(); net.SetInput(blob); net.Forward(outputBlobs, outputBlobNames); Mat scores = outputBlobs[0]; Mat geometry = outputBlobs[1]; // Decode predicted bounding boxes (decode the positions of the text boxes along with their orientation) Decode(scores, geometry, ConfThreshold, out var boxes, out var confidences); // Apply non-maximum suppression procedure for filtering out the false positives and get the final predictions CvDnn.NMSBoxes(boxes, confidences, ConfThreshold, NmsThreshold, out var indices); // Render detections. Point2f ratio = new Point2f((float)img.Cols / InputWidth, (float)img.Rows / InputHeight); for (var i = 0; i < indices.Length; ++i) { RotatedRect box = boxes[indices[i]]; Point2f[] vertices = box.Points(); for (int j = 0; j < 4; ++j) { vertices[j].X *= ratio.X; vertices[j].Y *= ratio.Y; } for (int j = 0; j < 4; ++j) { Cv2.Line(img, (int)vertices[j].X, (int)vertices[j].Y, (int)vertices[(j + 1) % 4].X, (int)vertices[(j + 1) % 4].Y, new Scalar(0, 255, 0), 3); } } ShowImagesWhenDebugMode(img); } }
private static (Mat, List <string>, List <Point>, List <float>, List <Rect2d>) GetResult(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold) { var classNames = new List <string>(); var centers = new List <Point>(); var classIds = new List <int>(); var confidences = new List <float>(); var probabilities = new List <float>(); var boxes = new List <Rect2d>(); var selectedBoxes = new List <Rect2d>(); var selectedConf = new List <float>(); var w = image.Width; var h = image.Height; foreach (var pred in output) { for (var i = 0; i < pred.Rows; i++) { var confidence = pred.At <float>(i, 4); if (confidence > threshold) { Cv2.MinMaxLoc(pred.Row(i).ColRange(5, pred.Cols), out _, out Point classIdPoint); var probability = pred.At <float>(i, classIdPoint.X + 5); if (probability > threshold) { var centerX = pred.At <float>(i, 0) * w; var centerY = pred.At <float>(i, 1) * h; var width = pred.At <float>(i, 2) * w; var height = pred.At <float>(i, 3) * h; classIds.Add(classIdPoint.X); confidences.Add(confidence); probabilities.Add(probability); boxes.Add(new Rect2d(centerX, centerY, width, height)); } } } } CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out int[] indices); foreach (var i in indices) { var box = boxes[i]; selectedBoxes.Add(box); selectedConf.Add(confidences[i]); classNames.Add(_labels[classIds[i]]); centers.Add(new Point(box.X, box.Y)); Draw(image, classIds[i], confidences[i], box.X, box.Y, box.Width, box.Height); } return(image, classNames, centers, selectedConf, selectedBoxes); }
public void NMSBoxes() { var bboxes = new Rect[] { new Rect(10, 10, 20, 20), new Rect(100, 100, 20, 20), new Rect(1000, 1000, 20, 20) }; var scores = new float[] { 1.0f, 0.1f, 0.6f }; float scoreThreshold = 0.5f; float nmsThreshold = 0.4f; CvDnn.NMSBoxes(bboxes, scores, scoreThreshold, nmsThreshold, out var indices); Assert.Equal(2, indices.Length); Assert.Equal(0, indices[0]); Assert.Equal(2, indices[1]); }
public IEnumerable <(float Confidence, double CenterX, double CenterY, double Width, double Height)> Detect(Mat image) { var blob = CvDnn.BlobFromImage(image, scalarFactor, configSize, blobFromImageMeanParams, true, false); net.SetInput(blob); net.Forward(outputLayers, outputNames); var items = new List <(float Confidence, double CenterX, double CenterY, double Width, double Height)>(); foreach (var prob in outputLayers) { for (var i = 0; i < prob.Rows; i++) { var confidence = prob.At <float>(i, 4); if (confidence > Defaults.ConfidenceThreshold) { Cv2.MinMaxLoc(prob.Row(i).ColRange(5, prob.Cols), out _, out Point max); var type = max.X; var label = labels[type]; var probability = prob.At <float>(i, type + 5); if (probability > Defaults.ConfidenceThreshold && label.Equals("person")) { var centerX = prob.At <float>(i, 0) * image.Width; var centerY = prob.At <float>(i, 1) * image.Height; var width = prob.At <float>(i, 2) * image.Width; var height = prob.At <float>(i, 3) * image.Height; items.Add((confidence, centerX, centerY, width, height)); } } } } //return items; CvDnn.NMSBoxes( items.Select(i => new Rect((int)(i.CenterX - (i.Width / 2)), (int)(i.CenterY - (i.Height / 2)), (int)i.Width, (int)i.Height)), items.Select(i => i.Confidence), (float)Defaults.ConfidenceThreshold, (float)Defaults.NonMaximaSupressionThreshold, out int[] indices); for (int i = 0; i < indices.Length; i++) { yield return(items[indices[i]]); } }
public IEnumerable <(float Confidence, double CenterX, double CenterY, double Width, double Height)> Detect(Mat image) { var items = new List <(float Confidence, double CenterX, double CenterY, double Width, double Height)>(); foreach (var item in yolo.Detect(image.ToBytes())) { if (item.Type.Equals("person")) { items.Add(((float)item.Confidence, item.X + item.Width / 2.0, item.Y + item.Height / 2.0, item.Width, item.Height)); } } //return items; CvDnn.NMSBoxes( items.Select(i => new Rect((int)(i.CenterX - (i.Width / 2)), (int)(i.CenterY - (i.Height / 2)), (int)i.Width, (int)i.Height)), items.Select(i => i.Confidence), (float)Defaults.ConfidenceThreshold, (float)Defaults.NonMaximaSupressionThreshold, out int[] indices); for (int i = 0; i < indices.Length; i++) { yield return(items[indices[i]]); } }
/// <summary> /// Get result form all output /// </summary> /// <param name="output"></param> /// <param name="image"></param> /// <param name="threshold"></param> /// <param name="nmsThreshold">threshold for nms</param> /// <param name="nms">Enable Non-maximum suppression or not</param> private static void GetResult(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold, bool nms = true) { //for nms var classIds = new List <int>(); var confidences = new List <float>(); var probabilities = new List <float>(); var boxes = new List <Rect2d>(); var w = image.Width; var h = image.Height; /* * YOLO3 COCO trainval output * 0 1 : center 2 3 : w/h * 4 : confidence 5 ~ 84 : class probability */ const int prefix = 5; //skip 0~4 foreach (var prob in output) { for (var i = 0; i < prob.Rows; i++) { var confidence = prob.At <float>(i, 4); if (confidence > threshold) { //get classes probability Cv2.MinMaxLoc(prob.Row(i).ColRange(prefix, prob.Cols), out _, out Point max); var classes = max.X; var probability = prob.At <float>(i, classes + prefix); if (probability > threshold) //more accuracy, you can cancel it { //get center and width/height var centerX = prob.At <float>(i, 0) * w; var centerY = prob.At <float>(i, 1) * h; var width = prob.At <float>(i, 2) * w; var height = prob.At <float>(i, 3) * h; if (!nms) { // draw result (if don't use NMSBoxes) Draw(image, classes, confidence, probability, centerX, centerY, width, height); continue; } //put data to list for NMSBoxes classIds.Add(classes); confidences.Add(confidence); probabilities.Add(probability); boxes.Add(new Rect2d(centerX, centerY, width, height)); } } } } if (!nms) { return; } //using non-maximum suppression to reduce overlapping low confidence box CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out int[] indices); Console.WriteLine($"NMSBoxes drop {confidences.Count - indices.Length} overlapping result."); foreach (var i in indices) { var box = boxes[i]; Draw(image, classIds[i], confidences[i], probabilities[i], box.X, box.Y, box.Width, box.Height); } }
public Rectangle[] Detect(Mat mat, byte[] buffer) { var matWidth = mat.Width; var matHeight = mat.Height; //using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(104, 117, 123), false, false); //using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(416, 416), mean: new OpenCvSharp.Scalar(0, 0, 0), swapRB: true, crop: false, ddepth: MatType.CV_8U); // , ddepth: MatType.CV_8U); //using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(416, 416), mean: new OpenCvSharp.Scalar(0, 0, 0), swapRB: true, crop: false, ddepth: MatType.CV_8U); // , ddepth: MatType.CV_8U); //Net.SetInput(blob, scaleFactor: 1.0 / 255.0); using var blob = CvDnn.BlobFromImage(mat, 1.0 / 255.0, new OpenCvSharp.Size(416, 416), crop: false); Net.SetInput(blob); var outputLayers = new Mat[OutputLayerNames.Length]; try { for (var i = 0; i < outputLayers.Length; i++) { outputLayers[i] = new Mat(); } Net.Forward(outputLayers, OutputLayerNames); //using var detections = Net.Forward() //using var detectionMat = new Mat(detections.Size(2), detections.Size(3), MatType.CV_32F, detections.Ptr(0)); var rects = new List <Rect>(); var confidences = new List <float>(); var rectangles = new List <Rectangle>(); for (var i = 0; i < outputLayers.Length; i++) { var outs = outputLayers[i]; for (var j = 0; j < outs.Rows; j++) { var cols = outs.Row(j).ColRange(5, outs.Cols); Cv2.MinMaxLoc(cols, out var _, out var confidence, out var _, out var maxVal); //Cv2.MinMaxLoc(cols, out var minVal, out OpenCvSharp.Point maxVal); var classes = maxVal.X; //var confidence = outs.At<float>(j, classes + 5); if (confidence > 0.5) { var centerX = outs.At <float>(j, 0) * matWidth; var centerY = outs.At <float>(j, 1) * matHeight; var width = outs.At <float>(j, 2) * matWidth; var height = outs.At <float>(j, 3) * matHeight; var left = centerX - (width / 2); var top = centerY - (height / 2); rects.Add(new Rect((int)left, (int)top, (int)width, (int)height)); confidences.Add((float)confidence); Console.WriteLine($"Class {classes} at {centerX},{centerY} of {width}x{height} with confidence {confidence}"); rectangles.Add(new Rectangle((int)(centerX - (width / 2)), (int)(centerY - (height / 2)), (int)(centerX + (width / 2)), (int)(centerY + (height / 2)))); } } } //for (var i = 0; i < detectionMat.Rows; i++) //{ // var confidence = detectionMat.At<float>(i, 2); // if (confidence > 0.7) // { // var left = (int)(detectionMat.At<float>(i, 3) * frameWidth); // var top = (int)(detectionMat.At<float>(i, 4) * frameHeight); // var right = (int)(detectionMat.At<float>(i, 5) * frameWidth); // var bottom = (int)(detectionMat.At<float>(i, 6) * frameHeight); // rectangles.Add(new Rectangle(left, top, right - left, bottom - top)); // } //} CvDnn.NMSBoxes(rects, confidences, 0.5f, 0.3f, out var indices); var results = new List <Rectangle>(indices.Length); foreach (var index in indices) { results.Add(rectangles[index]); } return(results.ToArray()); } finally { foreach (var outputLayer in outputLayers) { outputLayer?.Dispose(); } } }
private DnnDetectedObject[] ExtractYolo2Results(IEnumerable <Mat> output, Mat image, float threshold, float nmsThreshold, bool nms = true) { var classIds = new List <int>(); var confidences = new List <float>(); var probabilities = new List <float>(); var boxes = new List <Rect2d>(); var w = image.Width; var h = image.Height; var prob = output.First(); /* YOLO2 VOC output * 0 1 : center 2 3 : w/h * 4 : confidence 5 ~24 : class probability */ const int prefix = 5; //skip 0~4 for (int i = 0; i < prob.Rows; i++) { var confidence = prob.At <float>(i, 4); if (confidence > threshold) { //get classes probability Cv2.MinMaxLoc(prob.Row(i).ColRange(prefix, prob.Cols), out _, out Point max); var classes = max.X; var probability = prob.At <float>(i, classes + prefix); if (probability > threshold) //more accuracy { //get center and width/height var centerX = prob.At <float>(i, 0) * w; var centerY = prob.At <float>(i, 1) * h; var width = prob.At <float>(i, 2) * w; var height = prob.At <float>(i, 3) * h; float X = Math.Max(0, centerX - (width / 2.0f)); float Y = Math.Max(0, centerY - (height / 2.0f)); //put data to list for NMSBoxes classIds.Add(classes); confidences.Add(confidence); probabilities.Add(probability); boxes.Add(new Rect2d(X, Y, width, height)); } } } int[] indices; if (!nms) { //using non-maximum suppression to reduce overlapping low confidence box indices = Enumerable.Range(0, boxes.Count).ToArray(); } else { CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out indices); _logger.LogInformation($"NMSBoxes drop {confidences.Count - indices.Length} overlapping result."); } var result = new List <DnnDetectedObject>(); foreach (var i in indices) { var box = boxes[i]; var detection = new DnnDetectedObject() { Index = classIds[i], Label = Labels[classIds[i]], Color = Colors[classIds[i]], Probability = probabilities[i], BoundingBox = box }; result.Add(detection); } return(result.ToArray()); }
/// <summary> /// Get result form all output /// </summary> /// <param name="outputs"></param> /// <param name="image"></param> /// <param name="confidenceThreshold"></param> /// <param name="nmsThreshold">threshold for nms</param> /// <param name="nms">Enable Non-maximum suppression or not</param> private void DrawResults(IEnumerable <Mat> outputs, Mat image, float confidenceThreshold, float nmsThreshold, bool nms = true) { //for nms var classIds = new List <int>(); var confidences = new List <float>(); var probabilities = new List <float>(); var boxes = new List <Rect2d>(); var w = image.Width; var h = image.Height; /* * YOLO3 COCO trainval output * 0 1 : center * 2 3 : w/h * 4 : confidence * 5 ~ 84 : class probability */ const int prefix = 5; //skip 0~4 foreach (var output in outputs) { for (var i = 0; i < output.Rows; i++) { var confidence = output.At <float>(i, 4); if (confidence > confidenceThreshold) { //get classes probability Cv2.MinMaxLoc(output.Row(i).ColRange(prefix, output.Cols), out _, out double max); var classes = (int)max; var probability = output.At <float>(i, classes + prefix); if (probability > confidenceThreshold) //more accuracy, you can cancel it { //get center and width/height var centerX = output.At <float>(i, 0) * w; var centerY = output.At <float>(i, 1) * h; var width = output.At <float>(i, 2) * w; var height = output.At <float>(i, 3) * h; if (!nms) { // draw result (if don't use NMSBoxes) DrawResult(image, classes, confidence, probability, centerX, centerY, width, height); continue; } //put data to list for NMSBoxes classIds.Add(classes); confidences.Add(confidence); probabilities.Add(probability); boxes.Add(new Rect2d(centerX, centerY, width, height)); } } } } if (!nms) { return; } //using non-maximum suppression to reduce overlapping low confidence box CvDnn.NMSBoxes(boxes, confidences, confidenceThreshold, nmsThreshold, out int[] indices); foreach (var i in indices) { var box = boxes[i]; DrawResult(image, classIds[i], confidences[i], probabilities[i], box.X, box.Y, box.Width, box.Height); } }