private static void DrawMask(IInputOutputArray image, Mat mask, Rectangle rect, MCvScalar color) { using (Mat maskLarge = new Mat()) using (Mat maskLargeInv = new Mat()) using (InputArray iaImage = image.GetInputArray()) using (Mat matImage = iaImage.GetMat()) using (Mat subRegion = new Mat(matImage, rect)) using (Mat largeColor = new Mat( subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3)) { CvInvoke.Resize(mask, maskLarge, rect.Size); //give the mask at least 30% transparency using (ScalarArray sa = new ScalarArray(0.7)) CvInvoke.Min(sa, maskLarge, maskLarge); //Create the inverse mask for the original image using (ScalarArray sa = new ScalarArray(1.0)) CvInvoke.Subtract(sa, maskLarge, maskLargeInv); //The mask color largeColor.SetTo(color); if (subRegion.NumberOfChannels == 4) { using (Mat bgrSubRegion = new Mat()) { CvInvoke.CvtColor(subRegion, bgrSubRegion, ColorConversion.Bgra2Bgr); CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge, maskLargeInv, bgrSubRegion); CvInvoke.CvtColor(bgrSubRegion, subRegion, ColorConversion.Bgr2Bgra); } } else { CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion); } } }
public void Recognize(Mat m) { int[] dim = new int[] { 1, m.Height, m.Width, 3 }; if (_imageTensor == null) { _imageTensor = new Tensor(Emgu.TF.DataType.Uint8, dim); } else { if (!(_imageTensor.Type == Emgu.TF.DataType.Uint8 && Enumerable.SequenceEqual(dim, _imageTensor.Dim))) { _imageTensor.Dispose(); _imageTensor = new Tensor(Emgu.TF.DataType.Uint8, dim); } } Emgu.TF.TensorConvert.ReadTensorFromMatBgr(m, _imageTensor); MaskRcnnInceptionV2Coco.RecognitionResult[] results; if (_coldSession) { //First run of the recognition graph, here we will compile the graph and initialize the session //This is expected to take much longer time than consecutive runs. results = _inceptionGraph.Recognize(_imageTensor)[0]; _coldSession = false; } //Here we are trying to time the execution of the graph after it is loaded Stopwatch sw = Stopwatch.StartNew(); results = _inceptionGraph.Recognize(_imageTensor)[0]; sw.Stop(); int goodResultCount = 0; foreach (var r in results) { if (r.Probability > 0.5) { float x1 = r.Region[0] * m.Height; float y1 = r.Region[1] * m.Width; float x2 = r.Region[2] * m.Height; float y2 = r.Region[3] * m.Width; RectangleF rectf = new RectangleF(y1, x1, y2 - y1, x2 - x1); Rectangle rect = Rectangle.Round(rectf); rect.Intersect(new Rectangle(Point.Empty, m.Size)); //only keep the region that is inside the image if (rect.IsEmpty) { continue; } //draw the rectangle around the region CvInvoke.Rectangle(m, rect, new Emgu.CV.Structure.MCvScalar(0, 0, 255), 2); #region draw the mask float[,] mask = r.Mask; GCHandle handle = GCHandle.Alloc(mask, GCHandleType.Pinned); using (Mat mk = new Mat(new Size(mask.GetLength(1), mask.GetLength(0)), Emgu.CV.CvEnum.DepthType.Cv32F, 1, handle.AddrOfPinnedObject(), mask.GetLength(1) * sizeof(float))) using (Mat subRegion = new Mat(m, rect)) using (Mat maskLarge = new Mat()) using (Mat maskLargeInv = new Mat()) using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3)) { CvInvoke.Resize(mk, maskLarge, subRegion.Size); //give the mask at least 30% transparency using (ScalarArray sa = new ScalarArray(0.7)) CvInvoke.Min(sa, maskLarge, maskLarge); //Create the inverse mask for the original image using (ScalarArray sa = new ScalarArray(1.0)) CvInvoke.Subtract(sa, maskLarge, maskLargeInv); //The mask color largeColor.SetTo(new Emgu.CV.Structure.MCvScalar(255, 0, 0)); CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion); } handle.Free(); #endregion //draw the label CvInvoke.PutText(m, r.Label, Point.Round(rect.Location), Emgu.CV.CvEnum.FontFace.HersheyComplex, 1.0, new Emgu.CV.Structure.MCvScalar(0, 255, 0), 1); goodResultCount++; } } String resStr = String.Format("{0} objects detected in {1} milliseconds.", goodResultCount, sw.ElapsedMilliseconds); if (_renderMat == null) { _renderMat = new Mat(); } m.CopyTo(_renderMat); //Bitmap bmp = _renderMat.ToBitmap(); if (InvokeRequired) { this.Invoke((MethodInvoker)(() => { messageLabel.Text = resStr; pictureBox.Image = _renderMat; })); } else { messageLabel.Text = resStr; pictureBox.Image = _renderMat; } }
public DnnPage() : base() { var button = this.GetButton(); button.Text = "Perform Mask-rcnn Detection"; button.Clicked += OnButtonClicked; OnImagesLoaded += async(sender, image) => { if (image == null || image[0] == null) { return; } SetMessage("Please wait..."); SetImage(null); Task <Tuple <Mat, String, long> > t = new Task <Tuple <Mat, String, long> >( () => { InitDetector(); String msg = String.Empty; using (Mat blob = DnnInvoke.BlobFromImage(image[0])) using (VectorOfMat tensors = new VectorOfMat()) { _maskRcnnDetector.SetInput(blob, "image_tensor"); Stopwatch watch = Stopwatch.StartNew(); _maskRcnnDetector.Forward(tensors, new string[] { "detection_out_final", "detection_masks" }); watch.Stop(); msg = String.Format("Mask RCNN inception completed in {0} milliseconds.", watch.ElapsedMilliseconds); using (Mat boxes = tensors[0]) using (Mat masks = tensors[1]) { System.Drawing.Size imgSize = image[0].Size; float[,,,] boxesData = boxes.GetData(true) as float[, , , ]; int numDetections = boxesData.GetLength(2); for (int i = 0; i < numDetections; i++) { float score = boxesData[0, 0, i, 2]; if (score > 0.5) { int classId = (int)boxesData[0, 0, i, 1]; String label = _labels[classId]; MCvScalar color = _colors[classId]; float left = boxesData[0, 0, i, 3] * imgSize.Width; float top = boxesData[0, 0, i, 4] * imgSize.Height; float right = boxesData[0, 0, i, 5] * imgSize.Width; float bottom = boxesData[0, 0, i, 6] * imgSize.Height; RectangleF rectF = new RectangleF(left, top, right - left, bottom - top); Rectangle rect = Rectangle.Round(rectF); rect.Intersect(new Rectangle(Point.Empty, imgSize)); CvInvoke.Rectangle(image[0], rect, new MCvScalar(0, 0, 0, 0), 1); CvInvoke.PutText(image[0], label, rect.Location, FontFace.HersheyComplex, 1.0, new MCvScalar(0, 0, 255), 2); int[] masksDim = masks.SizeOfDimension; using (Mat mask = new Mat( masksDim[2], masksDim[3], DepthType.Cv32F, 1, masks.GetDataPointer(i, classId), masksDim[3] * masks.ElementSize)) using (Mat maskLarge = new Mat()) using (Mat maskLargeInv = new Mat()) using (Mat subRegion = new Mat(image[0], rect)) using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3)) { CvInvoke.Resize(mask, maskLarge, rect.Size); //give the mask at least 30% transparency using (ScalarArray sa = new ScalarArray(0.7)) CvInvoke.Min(sa, maskLarge, maskLarge); //Create the inverse mask for the original image using (ScalarArray sa = new ScalarArray(1.0)) CvInvoke.Subtract(sa, maskLarge, maskLargeInv); //The mask color largeColor.SetTo(color); if (subRegion.NumberOfChannels == 4) { using (Mat bgrSubRegion = new Mat()) { CvInvoke.CvtColor(subRegion, bgrSubRegion, ColorConversion.Bgra2Bgr); CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge, maskLargeInv, bgrSubRegion); CvInvoke.CvtColor(bgrSubRegion, subRegion, ColorConversion.Bgr2Bgra); } } else { CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion); } } } } } } long time = 0; return(new Tuple <Mat, String, long>(image[0], msg, time)); }); t.Start(); var result = await t; SetImage(t.Result.Item1); //String computeDevice = CvInvoke.UseOpenCL ? "OpenCL: " + Ocl.Device.Default.Name : "CPU"; SetMessage(t.Result.Item2); }; }
public DnnPage() : base() { var button = this.GetButton(); button.Text = "Perform Mask-rcnn Detection"; button.Clicked += OnButtonClicked; OnImagesLoaded += async(sender, image) => { if (image == null || image[0] == null) { return; } SetMessage("Please wait..."); SetImage(null); Task <Tuple <Mat, String, long> > t = new Task <Tuple <Mat, String, long> >( () => { String configFile = "mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"; #if __ANDROID__ String path = System.IO.Path.Combine(Android.OS.Environment.ExternalStorageDirectory.AbsolutePath, Android.OS.Environment.DirectoryDownloads, "dnn_data"); FileInfo configFileInfo = AndroidFileAsset.WritePermanantFileAsset(Android.App.Application.Context, configFile, "dnn_data", AndroidFileAsset.OverwriteMethod.AlwaysOverwrite); configFile = configFileInfo.FullName; #else String path = "./dnn_data/"; #endif String graphFile = DnnDownloadFile(path, "frozen_inference_graph.pb"); String lookupFile = DnnDownloadFile(path, "coco-labels-paper.txt"); string[] labels = File.ReadAllLines(lookupFile); Emgu.CV.Dnn.Net net = Emgu.CV.Dnn.DnnInvoke.ReadNetFromTensorflow(graphFile, configFile); Mat blob = DnnInvoke.BlobFromImage(image[0]); net.SetInput(blob, "image_tensor"); using (VectorOfMat tensors = new VectorOfMat()) { net.Forward(tensors, new string[] { "detection_out_final", "detection_masks" }); using (Mat boxes = tensors[0]) using (Mat masks = tensors[1]) { System.Drawing.Size imgSize = image[0].Size; float[,,,] boxesData = boxes.GetData(true) as float[, , , ]; //float[,,,] masksData = masks.GetData(true) as float[,,,]; int numDetections = boxesData.GetLength(2); for (int i = 0; i < numDetections; i++) { float score = boxesData[0, 0, i, 2]; if (score > 0.5) { int classId = (int)boxesData[0, 0, i, 1]; String label = labels[classId]; float left = boxesData[0, 0, i, 3] * imgSize.Width; float top = boxesData[0, 0, i, 4] * imgSize.Height; float right = boxesData[0, 0, i, 5] * imgSize.Width; float bottom = boxesData[0, 0, i, 6] * imgSize.Height; RectangleF rectF = new RectangleF(left, top, right - left, bottom - top); Rectangle rect = Rectangle.Round(rectF); rect.Intersect(new Rectangle(Point.Empty, imgSize)); CvInvoke.Rectangle(image[0], rect, new MCvScalar(0, 0, 0, 0), 1); CvInvoke.PutText(image[0], label, rect.Location, FontFace.HersheyComplex, 1.0, new MCvScalar(0, 0, 255), 2); int[] masksDim = masks.SizeOfDimension; using (Mat mask = new Mat( masksDim[2], masksDim[3], DepthType.Cv32F, 1, //masks.DataPointer + //(i * masksDim[1] + classId ) //* masksDim[2] * masksDim[3] * masks.ElementSize, masks.GetDataPointer(i, classId), masksDim[3] * masks.ElementSize)) using (Mat maskLarge = new Mat()) using (Mat maskLargeInv = new Mat()) using (Mat subRegion = new Mat(image[0], rect)) using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3)) { CvInvoke.Resize(mask, maskLarge, rect.Size); //give the mask at least 30% transparency using (ScalarArray sa = new ScalarArray(0.7)) CvInvoke.Min(sa, maskLarge, maskLarge); //Create the inverse mask for the original image using (ScalarArray sa = new ScalarArray(1.0)) CvInvoke.Subtract(sa, maskLarge, maskLargeInv); //The mask color largeColor.SetTo(new Emgu.CV.Structure.MCvScalar(255, 0, 0)); if (subRegion.NumberOfChannels == 4) { using (Mat bgrSubRegion = new Mat()) { CvInvoke.CvtColor(subRegion, bgrSubRegion, ColorConversion.Bgra2Bgr); CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge, maskLargeInv, bgrSubRegion); CvInvoke.CvtColor(bgrSubRegion, subRegion, ColorConversion.Bgr2Bgra); } } else { CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion); } } } } } } long time = 0; return(new Tuple <Mat, String, long>(image[0], null, time)); }); t.Start(); var result = await t; SetImage(t.Result.Item1); String computeDevice = CvInvoke.UseOpenCL ? "OpenCL: " + Ocl.Device.Default.Name : "CPU"; SetMessage(t.Result.Item2); }; }