private static void DrawMask(IInputOutputArray image, Mat mask, Rectangle rect, MCvScalar color)
{
using (Mat maskLarge = new Mat())
using (Mat maskLargeInv = new Mat())
using (InputArray iaImage = image.GetInputArray())
using (Mat matImage = iaImage.GetMat())
using (Mat subRegion = new Mat(matImage, rect))
using (Mat largeColor = new Mat(
subRegion.Size,
Emgu.CV.CvEnum.DepthType.Cv8U,
3))
{
CvInvoke.Resize(mask, maskLarge, rect.Size);
//give the mask at least 30% transparency
using (ScalarArray sa = new ScalarArray(0.7))
CvInvoke.Min(sa, maskLarge, maskLarge);
//Create the inverse mask for the original image
using (ScalarArray sa = new ScalarArray(1.0))
CvInvoke.Subtract(sa, maskLarge, maskLargeInv);
//The mask color
largeColor.SetTo(color);
if (subRegion.NumberOfChannels == 4)
{
using (Mat bgrSubRegion = new Mat())
{
CvInvoke.CvtColor(subRegion, bgrSubRegion,
ColorConversion.Bgra2Bgr);
CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge,
maskLargeInv, bgrSubRegion);
CvInvoke.CvtColor(bgrSubRegion, subRegion,
ColorConversion.Bgr2Bgra);
}
}
else
{
CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv,
subRegion);
}
}
}
public void Recognize(Mat m)
{
int[] dim = new int[] { 1, m.Height, m.Width, 3 };
if (_imageTensor == null)
{
_imageTensor = new Tensor(Emgu.TF.DataType.Uint8, dim);
}
else
{
if (!(_imageTensor.Type == Emgu.TF.DataType.Uint8 && Enumerable.SequenceEqual(dim, _imageTensor.Dim)))
{
_imageTensor.Dispose();
_imageTensor = new Tensor(Emgu.TF.DataType.Uint8, dim);
}
}
Emgu.TF.TensorConvert.ReadTensorFromMatBgr(m, _imageTensor);
MaskRcnnInceptionV2Coco.RecognitionResult[] results;
if (_coldSession)
{
//First run of the recognition graph, here we will compile the graph and initialize the session
//This is expected to take much longer time than consecutive runs.
results = _inceptionGraph.Recognize(_imageTensor)[0];
_coldSession = false;
}
//Here we are trying to time the execution of the graph after it is loaded
Stopwatch sw = Stopwatch.StartNew();
results = _inceptionGraph.Recognize(_imageTensor)[0];
sw.Stop();
int goodResultCount = 0;
foreach (var r in results)
{
if (r.Probability > 0.5)
{
float x1 = r.Region[0] * m.Height;
float y1 = r.Region[1] * m.Width;
float x2 = r.Region[2] * m.Height;
float y2 = r.Region[3] * m.Width;
RectangleF rectf = new RectangleF(y1, x1, y2 - y1, x2 - x1);
Rectangle rect = Rectangle.Round(rectf);
rect.Intersect(new Rectangle(Point.Empty, m.Size)); //only keep the region that is inside the image
if (rect.IsEmpty)
{
continue;
}
//draw the rectangle around the region
CvInvoke.Rectangle(m, rect, new Emgu.CV.Structure.MCvScalar(0, 0, 255), 2);
#region draw the mask
float[,] mask = r.Mask;
GCHandle handle = GCHandle.Alloc(mask, GCHandleType.Pinned);
using (Mat mk = new Mat(new Size(mask.GetLength(1), mask.GetLength(0)), Emgu.CV.CvEnum.DepthType.Cv32F, 1, handle.AddrOfPinnedObject(), mask.GetLength(1) * sizeof(float)))
using (Mat subRegion = new Mat(m, rect))
using (Mat maskLarge = new Mat())
using (Mat maskLargeInv = new Mat())
using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3))
{
CvInvoke.Resize(mk, maskLarge, subRegion.Size);
//give the mask at least 30% transparency
using (ScalarArray sa = new ScalarArray(0.7))
CvInvoke.Min(sa, maskLarge, maskLarge);
//Create the inverse mask for the original image
using (ScalarArray sa = new ScalarArray(1.0))
CvInvoke.Subtract(sa, maskLarge, maskLargeInv);
//The mask color
largeColor.SetTo(new Emgu.CV.Structure.MCvScalar(255, 0, 0));
CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion);
}
handle.Free();
#endregion
//draw the label
CvInvoke.PutText(m, r.Label, Point.Round(rect.Location), Emgu.CV.CvEnum.FontFace.HersheyComplex, 1.0, new Emgu.CV.Structure.MCvScalar(0, 255, 0), 1);
goodResultCount++;
}
}
String resStr = String.Format("{0} objects detected in {1} milliseconds.", goodResultCount, sw.ElapsedMilliseconds);
if (_renderMat == null)
{
_renderMat = new Mat();
}
m.CopyTo(_renderMat);
//Bitmap bmp = _renderMat.ToBitmap();
if (InvokeRequired)
{
this.Invoke((MethodInvoker)(() =>
{
messageLabel.Text = resStr;
pictureBox.Image = _renderMat;
}));
}
else
{
messageLabel.Text = resStr;
pictureBox.Image = _renderMat;
}
}
public DnnPage()
: base()
{
var button = this.GetButton();
button.Text = "Perform Mask-rcnn Detection";
button.Clicked += OnButtonClicked;
OnImagesLoaded += async(sender, image) =>
{
if (image == null || image[0] == null)
{
return;
}
SetMessage("Please wait...");
SetImage(null);
Task <Tuple <Mat, String, long> > t = new Task <Tuple <Mat, String, long> >(
() =>
{
InitDetector();
String msg = String.Empty;
using (Mat blob = DnnInvoke.BlobFromImage(image[0]))
using (VectorOfMat tensors = new VectorOfMat())
{
_maskRcnnDetector.SetInput(blob, "image_tensor");
Stopwatch watch = Stopwatch.StartNew();
_maskRcnnDetector.Forward(tensors, new string[] { "detection_out_final", "detection_masks" });
watch.Stop();
msg = String.Format("Mask RCNN inception completed in {0} milliseconds.",
watch.ElapsedMilliseconds);
using (Mat boxes = tensors[0])
using (Mat masks = tensors[1])
{
System.Drawing.Size imgSize = image[0].Size;
float[,,,] boxesData = boxes.GetData(true) as float[, , , ];
int numDetections = boxesData.GetLength(2);
for (int i = 0; i < numDetections; i++)
{
float score = boxesData[0, 0, i, 2];
if (score > 0.5)
{
int classId = (int)boxesData[0, 0, i, 1];
String label = _labels[classId];
MCvScalar color = _colors[classId];
float left = boxesData[0, 0, i, 3] * imgSize.Width;
float top = boxesData[0, 0, i, 4] * imgSize.Height;
float right = boxesData[0, 0, i, 5] * imgSize.Width;
float bottom = boxesData[0, 0, i, 6] * imgSize.Height;
RectangleF rectF = new RectangleF(left, top, right - left, bottom - top);
Rectangle rect = Rectangle.Round(rectF);
rect.Intersect(new Rectangle(Point.Empty, imgSize));
CvInvoke.Rectangle(image[0], rect, new MCvScalar(0, 0, 0, 0), 1);
CvInvoke.PutText(image[0], label, rect.Location, FontFace.HersheyComplex, 1.0,
new MCvScalar(0, 0, 255), 2);
int[] masksDim = masks.SizeOfDimension;
using (Mat mask = new Mat(
masksDim[2],
masksDim[3],
DepthType.Cv32F,
1,
masks.GetDataPointer(i, classId),
masksDim[3] * masks.ElementSize))
using (Mat maskLarge = new Mat())
using (Mat maskLargeInv = new Mat())
using (Mat subRegion = new Mat(image[0], rect))
using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3))
{
CvInvoke.Resize(mask, maskLarge, rect.Size);
//give the mask at least 30% transparency
using (ScalarArray sa = new ScalarArray(0.7))
CvInvoke.Min(sa, maskLarge, maskLarge);
//Create the inverse mask for the original image
using (ScalarArray sa = new ScalarArray(1.0))
CvInvoke.Subtract(sa, maskLarge, maskLargeInv);
//The mask color
largeColor.SetTo(color);
if (subRegion.NumberOfChannels == 4)
{
using (Mat bgrSubRegion = new Mat())
{
CvInvoke.CvtColor(subRegion, bgrSubRegion, ColorConversion.Bgra2Bgr);
CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge, maskLargeInv, bgrSubRegion);
CvInvoke.CvtColor(bgrSubRegion, subRegion, ColorConversion.Bgr2Bgra);
}
}
else
{
CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion);
}
}
}
}
}
}
long time = 0;
return(new Tuple <Mat, String, long>(image[0], msg, time));
});
t.Start();
var result = await t;
SetImage(t.Result.Item1);
//String computeDevice = CvInvoke.UseOpenCL ? "OpenCL: " + Ocl.Device.Default.Name : "CPU";
SetMessage(t.Result.Item2);
};
}
public DnnPage()
: base()
{
var button = this.GetButton();
button.Text = "Perform Mask-rcnn Detection";
button.Clicked += OnButtonClicked;
OnImagesLoaded += async(sender, image) =>
{
if (image == null || image[0] == null)
{
return;
}
SetMessage("Please wait...");
SetImage(null);
Task <Tuple <Mat, String, long> > t = new Task <Tuple <Mat, String, long> >(
() =>
{
String configFile = "mask_rcnn_inception_v2_coco_2018_01_28.pbtxt";
#if __ANDROID__
String path = System.IO.Path.Combine(Android.OS.Environment.ExternalStorageDirectory.AbsolutePath,
Android.OS.Environment.DirectoryDownloads, "dnn_data");
FileInfo configFileInfo = AndroidFileAsset.WritePermanantFileAsset(Android.App.Application.Context, configFile, "dnn_data", AndroidFileAsset.OverwriteMethod.AlwaysOverwrite);
configFile = configFileInfo.FullName;
#else
String path = "./dnn_data/";
#endif
String graphFile = DnnDownloadFile(path, "frozen_inference_graph.pb");
String lookupFile = DnnDownloadFile(path, "coco-labels-paper.txt");
string[] labels = File.ReadAllLines(lookupFile);
Emgu.CV.Dnn.Net net = Emgu.CV.Dnn.DnnInvoke.ReadNetFromTensorflow(graphFile, configFile);
Mat blob = DnnInvoke.BlobFromImage(image[0]);
net.SetInput(blob, "image_tensor");
using (VectorOfMat tensors = new VectorOfMat())
{
net.Forward(tensors, new string[] { "detection_out_final", "detection_masks" });
using (Mat boxes = tensors[0])
using (Mat masks = tensors[1])
{
System.Drawing.Size imgSize = image[0].Size;
float[,,,] boxesData = boxes.GetData(true) as float[, , , ];
//float[,,,] masksData = masks.GetData(true) as float[,,,];
int numDetections = boxesData.GetLength(2);
for (int i = 0; i < numDetections; i++)
{
float score = boxesData[0, 0, i, 2];
if (score > 0.5)
{
int classId = (int)boxesData[0, 0, i, 1];
String label = labels[classId];
float left = boxesData[0, 0, i, 3] * imgSize.Width;
float top = boxesData[0, 0, i, 4] * imgSize.Height;
float right = boxesData[0, 0, i, 5] * imgSize.Width;
float bottom = boxesData[0, 0, i, 6] * imgSize.Height;
RectangleF rectF = new RectangleF(left, top, right - left, bottom - top);
Rectangle rect = Rectangle.Round(rectF);
rect.Intersect(new Rectangle(Point.Empty, imgSize));
CvInvoke.Rectangle(image[0], rect, new MCvScalar(0, 0, 0, 0), 1);
CvInvoke.PutText(image[0], label, rect.Location, FontFace.HersheyComplex, 1.0,
new MCvScalar(0, 0, 255), 2);
int[] masksDim = masks.SizeOfDimension;
using (Mat mask = new Mat(
masksDim[2],
masksDim[3],
DepthType.Cv32F,
1,
//masks.DataPointer +
//(i * masksDim[1] + classId )
//* masksDim[2] * masksDim[3] * masks.ElementSize,
masks.GetDataPointer(i, classId),
masksDim[3] * masks.ElementSize))
using (Mat maskLarge = new Mat())
using (Mat maskLargeInv = new Mat())
using (Mat subRegion = new Mat(image[0], rect))
using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3))
{
CvInvoke.Resize(mask, maskLarge, rect.Size);
//give the mask at least 30% transparency
using (ScalarArray sa = new ScalarArray(0.7))
CvInvoke.Min(sa, maskLarge, maskLarge);
//Create the inverse mask for the original image
using (ScalarArray sa = new ScalarArray(1.0))
CvInvoke.Subtract(sa, maskLarge, maskLargeInv);
//The mask color
largeColor.SetTo(new Emgu.CV.Structure.MCvScalar(255, 0, 0));
if (subRegion.NumberOfChannels == 4)
{
using (Mat bgrSubRegion = new Mat())
{
CvInvoke.CvtColor(subRegion, bgrSubRegion, ColorConversion.Bgra2Bgr);
CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge, maskLargeInv, bgrSubRegion);
CvInvoke.CvtColor(bgrSubRegion, subRegion, ColorConversion.Bgr2Bgra);
}
}
else
{
CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion);
}
}
}
}
}
}
long time = 0;
return(new Tuple <Mat, String, long>(image[0], null, time));
});
t.Start();
var result = await t;
SetImage(t.Result.Item1);
String computeDevice = CvInvoke.UseOpenCL ? "OpenCL: " + Ocl.Device.Default.Name : "CPU";
SetMessage(t.Result.Item2);
};
}
