public static void Detect(Mat image, List <Rectangle> faces)
{
Net net = DnnInvoke.ReadNetFromTensorflow(@"C:\models\opencv_face_detector_uint8.pb", @"C:\models\opencv_face_detector.pbtxt");
Mat inputBlob = DnnInvoke.BlobFromImage(image, 1.0, new Size(300, 300), new MCvScalar(104.0, 117.0, 123.0), true, false);
net.SetInput(inputBlob, "data");
Mat detection = net.Forward("detection_out");
int resultRows = detection.SizeOfDimension[2];
int resultCols = detection.SizeOfDimension[3];
float[] temp = new float[resultRows * resultCols];
Marshal.Copy(detection.DataPointer, temp, 0, temp.Length);
for (int i = 0; i < resultRows; i++)
{
float confidence = temp[i * resultCols + 2];
if (confidence > 0.7)
{
int x1 = (int)(temp[i * resultCols + 3] * image.Width);
int y1 = (int)(temp[i * resultCols + 4] * image.Height);
int x2 = (int)(temp[i * resultCols + 5] * image.Width);
int y2 = (int)(temp[i * resultCols + 6] * image.Height);
Rectangle rectangle = new Rectangle(x1, y1, x2 - x1, y2 - y1);
faces.Add(rectangle);
}
}
}
public Mat Detect(Mat mat)
{
byte[] array = new byte[_width * _height * mat.ElemSize()];
Marshal.Copy(mat.DataStart, array, 0, array.Length);
using (Image <Bgr, byte> image1 = new Image <Bgr, byte>(_width, _height))
{
image1.Bytes = array;
var frame = image1.Mat;
int cols = 640;
int rows = 480;
_net.SetInput(DnnInvoke.BlobFromImage(frame, 1, new System.Drawing.Size(300, 300), default(MCvScalar), false, false));
using (Emgu.CV.Mat matt = _net.Forward())
{
float[,,,] flt = (float[, , , ])matt.GetData();
for (int x = 0; x < flt.GetLength(2); x++)
{
if (flt[0, 0, x, 2] > _probability)
{
int X1 = Convert.ToInt32(flt[0, 0, x, 3] * cols);
int Y1 = Convert.ToInt32(flt[0, 0, x, 4] * rows);
int X2 = Convert.ToInt32(flt[0, 0, x, 5] * cols);
int Y2 = Convert.ToInt32(flt[0, 0, x, 6] * rows);
mat.Rectangle(new OpenCvSharp.Rect((int)X1, (int)Y1, (int)X2 - (int)X1, (int)Y2 - (int)Y1), Scalar.Red);
}
}
}
}
return(mat);
}
/// <summary>
/// detect face and crop, otherwise returns null, option2
/// </summary>
/// <param name="image"></param>
/// <returns></returns>
private Mat GetDetectedFaceBoxSecond(Mat image)
{
var(h, w) = (image.Size.Height, image.Size.Width);
var resizedThreeHundred = new Mat();
var image600 = new Mat();
CvInvoke.Resize(image, image600, new Size(600, 600 * image.Height / image.Width));
CvInvoke.ResizeForFrame(image600, resizedThreeHundred, new Size(300, 300));
var blobFromImage = DnnInvoke.BlobFromImage(
resizedThreeHundred, 1, new Size(300, 300), new MCvScalar(104.0, 177.0, 123.0));
detector.SetInput(blobFromImage);
var detection = detector.Forward();
var data = (float[, , , ])detection.GetData();
//ensure at least one face was found
if (data.Length == 0)
{
return(null);
}
//we're making the assumption that each image has only ONE
//face, so find the bounding box with the largest probability
var maxi = GetMaxConfidenceIdx(detection, data);
var confidence = data[0, 0, maxi, 2];
if (confidence < minConfidence)
{
return(null);
}
//compute the (x, y)-coordinates of the bounding box for the face
var(startX, startY, endX, endY) = (data[0, 0, maxi, 3] * w, data[0, 0, maxi, 4] * h,
data[0, 0, maxi, 5] * w, data[0, 0, maxi, 6] * h);
// correct coordinates
if (startX < 0)
{
startX = 0;
}
if (endX > image.Width)
{
endX = image.Width;
}
if (startY < 0)
{
startY = 0;
}
if (endY > image.Height)
{
endY = image.Height;
}
//extract the face ROI and grab the ROI dimensions
var faceRect = Rectangle.FromLTRB((int)startX, (int)startY, (int)endX, (int)endY);
return(new Mat(resizedThreeHundred, faceRect));
}
/// <summary>
/// Receve an image from camera.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void Camera_ImageGrabbed(object sender, EventArgs e)
{
camera.Retrieve(frame);
Mat blobs = DnnInvoke.BlobFromImage(frame, 1.0, new System.Drawing.Size(detectionSize, detectionSize), swapRB: true);
net.SetInput(blobs);
Mat outp = net.Forward();
float[,,,] boxes = outp.GetData() as float[, , , ];
for (int i = 0; i < boxes.GetLength(2); i++)
{
int classID = Convert.ToInt32(boxes[0, 0, i, 1]);
float confidence = Convert.ToSingle(
boxes[0, 0, i, 2].ToString().Replace(",", "."), CultureInfo.InvariantCulture);
if (confidence < 0.6)
{
continue;
}
float Xstart = Convert.ToSingle(
boxes[0, 0, i, 3].ToString().Replace(",", "."), CultureInfo.InvariantCulture) * resolutionX;
float Ystart = Convert.ToSingle(
boxes[0, 0, i, 4].ToString().Replace(",", "."), CultureInfo.InvariantCulture) * resolutionY;
float Xend = Convert.ToSingle(
boxes[0, 0, i, 5].ToString().Replace(",", "."), CultureInfo.InvariantCulture) * resolutionX;
float Yend = Convert.ToSingle(
boxes[0, 0, i, 6].ToString().Replace(",", "."), CultureInfo.InvariantCulture) * resolutionY;
System.Drawing.Rectangle rect = new System.Drawing.Rectangle
{
X = (int)Xstart,
Y = (int)Ystart,
Height = (int)(Yend - Ystart),
Width = (int)(Xend - Xstart)
};
string label = labels[classID - 1];
frame.Draw(rect, new Bgr(0, 255, 0), 2);
frame.Draw(new System.Drawing.Rectangle((int)Xstart,
(int)Ystart - 35, label.Length * 18, 35), new Bgr(0, 255, 0), -1);
CvInvoke.PutText(frame, label, new System.Drawing.Point((int)Xstart,
(int)Ystart - 10), FontFace.HersheySimplex, 1.0, new MCvScalar(0, 0, 0), 2);
}
Dispatcher.Invoke(new Action(() =>
{
img.Source = frame.Bitmap.BitmapToBitmapSource();
}));
}
public void Detect(Mat image, List <Rectangle> fullFaceRegions, List <Rectangle> partialFaceRegions)
{
int imgDim = 300;
MCvScalar meanVal = new MCvScalar(104, 177, 123);
Size imageSize = image.Size;
using (Mat inputBlob = DnnInvoke.BlobFromImage(
image,
1.0,
new Size(imgDim, imgDim),
meanVal,
false,
false))
_faceDetector.SetInput(inputBlob, "data");
using (Mat detection = _faceDetector.Forward("detection_out"))
{
float confidenceThreshold = 0.5f;
//List<Rectangle> fullFaceRegions = new List<Rectangle>();
//List<Rectangle> partialFaceRegions = new List<Rectangle>();
Rectangle imageRegion = new Rectangle(Point.Empty, image.Size);
float[,,,] values = detection.GetData(true) as float[, , , ];
for (int i = 0; i < values.GetLength(2); i++)
{
float confident = values[0, 0, i, 2];
if (confident > confidenceThreshold)
{
float xLeftBottom = values[0, 0, i, 3] * imageSize.Width;
float yLeftBottom = values[0, 0, i, 4] * imageSize.Height;
float xRightTop = values[0, 0, i, 5] * imageSize.Width;
float yRightTop = values[0, 0, i, 6] * imageSize.Height;
RectangleF objectRegion = new RectangleF(
xLeftBottom,
yLeftBottom,
xRightTop - xLeftBottom,
yRightTop - yLeftBottom);
Rectangle faceRegion = Rectangle.Round(objectRegion);
if (imageRegion.Contains(faceRegion))
{
fullFaceRegions.Add(faceRegion);
}
else
{
partialFaceRegions.Add(faceRegion);
}
}
}
}
}
private void button2_Click(object sender, EventArgs e)
{
try
{
if (pictureBox1.Image == null)
{
throw new Exception("Draw a digit.");
}
if (model == null)
{
throw new Exception("Load the ONNX model.");
}
Bitmap bm = new Bitmap(pictureBox1.ClientSize.Width, pictureBox1.ClientSize.Height);
pictureBox1.DrawToBitmap(bm, pictureBox1.ClientRectangle);
var img = bm.ToImage <Gray, byte>()
.Not()
.SmoothGaussian(3)
.Resize(28, 28, Emgu.CV.CvEnum.Inter.Cubic)
.Mul(1 / 255.0f);
var input = DnnInvoke.BlobFromImage(img);
model.SetInput(input);
var output = model.Forward();
float[] array = new float[10];
output.CopyTo(array);
var prob = SoftMax(array);
int index = Array.IndexOf(prob, prob.Max());
lblDigit.Text = index.ToString();
chart1.Series.Clear();
chart1.Titles.Clear();
chart1.Series.Add("Hist");
chart1.Titles.Add("Probabilities");
for (int i = 0; i < prob.Length; i++)
{
chart1.Series["Hist"].Points.AddXY(i, prob[i]);
}
//pictureBox1.Image = img.AsBitmap();
}
catch (Exception ex)
{
lblMessage.Text = ex.Message;
}
}
/***** 检测手部 并画框 ****/
private Image <Bgr, byte> ProcessImageUseDnn(Image <Bgr, byte> img)
{
Mat frame = img.Mat; //获取输入图像的矩阵
Mat inputBlob = DnnInvoke.BlobFromImage(frame, 1, new Size(300, 300)); //转换为神经网络的输入格式 (300x300)
net.SetInput(inputBlob, "image_tensor"); //输入数据
Mat prob = net.Forward("detection_out"); //获取输出
byte[] data = new byte[5600]; //将输出从矩阵转换为数组, 便于处理
prob.CopyTo(data);
for (int i = 0; i < prob.SizeOfDimemsion[2]; i++) //遍历检测框
{
var d = BitConverter.ToSingle(data, i * 28 + 8); //获取检测框置信度
if (d > 0.3) //如果置信度大于30%, 则认为有效
{
var idx = (int)BitConverter.ToSingle(data, i * 28 + 4); //输出的类别, 这里只有hand一个类别
var w1 = (int)(BitConverter.ToSingle(data, i * 28 + 12) * img.Width); //检测框的位置 - 左侧
var h1 = (int)(BitConverter.ToSingle(data, i * 28 + 16) * img.Height); //检测框的位置 - 顶部
var w2 = (int)(BitConverter.ToSingle(data, i * 28 + 20) * img.Width); //检测框的位置 - 右侧
var h2 = (int)(BitConverter.ToSingle(data, i * 28 + 24) * img.Height); //检测框的位置 - 底部
double X = (w1 + w2) / 2; //检测框的位置 - 中心横坐标
double Y = (h1 + h2) / 2; //检测框的位置 - 中心纵坐标
AddPoint(X, Y); //调用添加检测点方法
if (!showBoxs) //如果没有开启检测框绘制功能, 则跳过绘制检测框部分
{
continue;
}
var label = $"hand {d * 100:0.00}%"; //标签文本: "类别(hand) + 置信度"
CvInvoke.Rectangle(img, new Rectangle(w1, h1, w2 - w1, h2 - h1), new MCvScalar(0, 255, 0), 2); //新建一个矩形对象来表示检测框
int baseline = 0;
var textSize = CvInvoke.GetTextSize(label, FontFace.HersheyTriplex, 0.5, 1, ref baseline); //计算标签大小
var y = h1 - textSize.Height < 0 ? h1 + textSize.Height : h1; //计算标签位置
CvInvoke.Rectangle(img, new Rectangle(w1, y - textSize.Height, textSize.Width, textSize.Height), new MCvScalar(0, 255, 0), -1); //绘制检测框
CvInvoke.PutText(img, label, new Point(w1, y), FontFace.HersheyTriplex, 0.5, new Bgr(0, 0, 0).MCvScalar); //绘制标签
}
}
UpdateTTL(); //调用更新生存时间方法
return(img);
}
/// <summary>
/// 偵測圖片中的臉部
/// </summary>
/// <param name="image"> 圖片 </param>
/// <returns> 臉部座標 (X, Y, 寬度, 高度) </returns>
public IEnumerable <IReadOnlyDictionary <string, int> > Detect(Mat image)
{
Mat inputBlob = DnnInvoke.BlobFromImage(image, 1.0, new Size(300, 300), new MCvScalar(104.0, 117.0, 123.0), true, false);
detectorModel.SetInput(inputBlob, "data");
Mat detection = detectorModel.Forward("detection_out");
// 神經網路輸出形狀 (1, 1, {face count}, 7)
// 第四項內容是人臉偵測結果的值共七個[0, 1, 2, 3, 4, 5, 6, 7]
// 其中第三個值(2的位置)：人臉偵測的信任分數，越高表示越像人臉
// 第四~七的值(3,4,5,6的位置)分別代表人臉的左上角(x, y)到右下角(x, y) 位置的比例
int resultFaceCount = detection.SizeOfDimension[2];
int resultFaceInfoLength = detection.SizeOfDimension[3];
float[] facesInfo = new float[resultFaceCount * resultFaceInfoLength];
Marshal.Copy(detection.DataPointer, facesInfo, 0, facesInfo.Length);
ConcurrentStack <IReadOnlyDictionary <string, int> > result = new ConcurrentStack <IReadOnlyDictionary <string, int> >();
Parallel.For(0, resultFaceCount,
faceIndex => {
float faceConfidence = facesInfo[faceIndex * resultFaceInfoLength + 2];
int x1 = Convert.ToInt32(facesInfo[faceIndex * resultFaceInfoLength + 3] * image.Width);
int y1 = Convert.ToInt32(facesInfo[faceIndex * resultFaceInfoLength + 4] * image.Height);
int x2 = Convert.ToInt32(facesInfo[faceIndex * resultFaceInfoLength + 5] * image.Width);
int y2 = Convert.ToInt32(facesInfo[faceIndex * resultFaceInfoLength + 6] * image.Height);
if (faceConfidence > 0.7 &&
x1 < image.Width && x2 <= image.Width &&
y1 < image.Height && y2 <= image.Height)
{
Dictionary <string, int> face = new Dictionary <string, int>
{
{ "top", x1 },
{ "left", y1 },
{ "width", x2 - x1 },
{ "height", y2 - y1 },
{ "x1", x1 },
{ "x2", x2 },
{ "y1", y1 },
{ "y2", y2 }
};
result.Push(face);
}
});
return(result);
}
// Ищем лица по списку изображений (SSD)
public List <int[][]> DetectFacesSDD(List <string> imagePaths)
{
List <int[][]> allFaces = new List <int[][]>()
{
};
int count = 0;
// Ищем лица для каждого изображения
foreach (var file in imagePaths)
{
List <int[]> faces = new List <int[]>();
int i = 0;
using (Image <Bgr, byte> image = new Image <Bgr, byte>(file))
{
int cols = image.Width;
int rows = image.Height;
Net net = DnnInvoke.ReadNetFromTensorflow(_modelFile, _configFile);
net.SetInput(DnnInvoke.BlobFromImage(image.Mat, 1, new System.Drawing.Size(300, 300), default(MCvScalar), false, false));
Mat mat = net.Forward();
float[,,,] flt = (float[, , , ])mat.GetData();
for (int x = 0; x < flt.GetLength(2); x++)
{
if (flt[0, 0, x, 2] > 0.2)
{
int left = Convert.ToInt32(flt[0, 0, x, 3] * cols);
int top = Convert.ToInt32(flt[0, 0, x, 4] * rows);
int right = Convert.ToInt32(flt[0, 0, x, 5] * cols) - left;
int bottom = Convert.ToInt32(flt[0, 0, x, 6] * rows) - top;
int[] face = new[] { left, top, right, bottom };
faces.Add(face);
i++;
}
}
}
allFaces.Add(faces.ToArray());
Console.WriteLine(count);
count++;
}
return(allFaces);
}
public Mat GetFaceEmbedding(Mat image, int detectAndCropMethod = 1) // image should be a cropped face image
{
Mat face;
switch (detectAndCropMethod)
{
case 1:
face = GetDetectedFaceBox(image);
break;
case 2:
face = GetDetectedFaceBoxSecond(image);
break;
default:
face = image; // the image has already detected and cropped
break;
}
if (face == null)
{
return(null);
}
//#load the image, resize it to have a width of 300 pixels (while
//# maintaining the aspect ratio), and then grab the image dimension
#if DEBUG
//DebugHelper.DrawFaceRectAndSave(image.Bitmap, faceRect);
#endif
var face96 = new Mat();
CvInvoke.ResizeForFrame(face, face96, new Size(96, 96));
var(fH, fW) = (face.Size.Height, face.Size.Width);
if (fH < 96 || fW < 96)
{
return(null);
}
//construct a blob for the face ROI, then pass the blob
//through our face embedding model to obtain the 128-d
//quantification of the face
var faceBlob = DnnInvoke.BlobFromImage(face96, 1.0 / 255, new Size(96, 96), new MCvScalar(0, 0, 0),
swapRB: true, crop: false);
embedder.SetInput(faceBlob);
var vec = embedder.Forward().Clone();
return(vec.Reshape(1));
}
public Mat GetFaceEmbeddingSecond(Mat image)
{
var(fH, fW) = (image.Size.Height, image.Size.Width);
if (fH < 300 || fW < 300)
{
return(null);
}
var face300 = new Mat();
CvInvoke.ResizeForFrame(image, face300, new Size(300, 300));
var faceBlob = DnnInvoke.BlobFromImage(face300, 1.0 / 255, new Size(300, 300), new MCvScalar(0, 0, 0),
swapRB: true, crop: false);
embedderSecond.SetInput(faceBlob);
var vec = embedderSecond.Forward().Clone();
return(vec.Reshape(1));
}
/// <summary>
/// Receive an image from camera.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void Camera_ImageGrabbed(object sender, EventArgs e)
{
camera.Retrieve(frame);
//CvInvoke.Flip(frame, frame, Emgu.CV.CvEnum.FlipType.Horizontal);
Mat blobs = DnnInvoke.BlobFromImage(frame, 1.0, new System.Drawing.Size(detectionSize, detectionSize));
net.SetInput(blobs);
Mat detections = net.Forward();
float[,,,] detectionsArrayInFloats = detections.GetData() as float[, , , ];
for (int i = 0; i < detectionsArrayInFloats.GetLength(2); i++)
{
if (Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 2], CultureInfo.InvariantCulture) > 0.4)
{
float Xstart = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 3],
CultureInfo.InvariantCulture) * detectionSize * xRate;
float Ystart = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 4],
CultureInfo.InvariantCulture) * detectionSize * yRate;
float Xend = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 5],
CultureInfo.InvariantCulture) * detectionSize * xRate;
float Yend = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 6],
CultureInfo.InvariantCulture) * detectionSize * yRate;
System.Drawing.Rectangle rect = new System.Drawing.Rectangle
{
X = (int)Xstart,
Y = (int)Ystart,
Height = (int)(Yend - Ystart),
Width = (int)(Xend - Xstart)
};
frame.Draw(rect, new Bgr(0, 255, 0), 2);
}
}
Dispatcher.Invoke(new Action(() =>
{
img.Source = frame.Bitmap.BitmapToBitmapSource();
}));
}
public DnnPage()
: base()
{
var button = this.GetButton();
button.Text = "Perform Mask-rcnn Detection";
button.Clicked += OnButtonClicked;
OnImagesLoaded += async(sender, image) =>
{
if (image == null || image[0] == null)
{
return;
}
SetMessage("Please wait...");
SetImage(null);
Task <Tuple <Mat, String, long> > t = new Task <Tuple <Mat, String, long> >(
() =>
{
String configFile = "mask_rcnn_inception_v2_coco_2018_01_28.pbtxt";
#if __ANDROID__
String path = System.IO.Path.Combine(Android.OS.Environment.ExternalStorageDirectory.AbsolutePath,
Android.OS.Environment.DirectoryDownloads, "dnn_data");
FileInfo configFileInfo = AndroidFileAsset.WritePermanantFileAsset(Android.App.Application.Context, configFile, "dnn_data", AndroidFileAsset.OverwriteMethod.AlwaysOverwrite);
configFile = configFileInfo.FullName;
#else
String path = "./dnn_data/";
#endif
String graphFile = DnnDownloadFile(path, "frozen_inference_graph.pb");
String lookupFile = DnnDownloadFile(path, "coco-labels-paper.txt");
string[] labels = File.ReadAllLines(lookupFile);
Emgu.CV.Dnn.Net net = Emgu.CV.Dnn.DnnInvoke.ReadNetFromTensorflow(graphFile, configFile);
Mat blob = DnnInvoke.BlobFromImage(image[0]);
net.SetInput(blob, "image_tensor");
using (VectorOfMat tensors = new VectorOfMat())
{
net.Forward(tensors, new string[] { "detection_out_final", "detection_masks" });
using (Mat boxes = tensors[0])
using (Mat masks = tensors[1])
{
System.Drawing.Size imgSize = image[0].Size;
float[,,,] boxesData = boxes.GetData(true) as float[, , , ];
//float[,,,] masksData = masks.GetData(true) as float[,,,];
int numDetections = boxesData.GetLength(2);
for (int i = 0; i < numDetections; i++)
{
float score = boxesData[0, 0, i, 2];
if (score > 0.5)
{
int classId = (int)boxesData[0, 0, i, 1];
String label = labels[classId];
float left = boxesData[0, 0, i, 3] * imgSize.Width;
float top = boxesData[0, 0, i, 4] * imgSize.Height;
float right = boxesData[0, 0, i, 5] * imgSize.Width;
float bottom = boxesData[0, 0, i, 6] * imgSize.Height;
RectangleF rectF = new RectangleF(left, top, right - left, bottom - top);
Rectangle rect = Rectangle.Round(rectF);
rect.Intersect(new Rectangle(Point.Empty, imgSize));
CvInvoke.Rectangle(image[0], rect, new MCvScalar(0, 0, 0, 0), 1);
CvInvoke.PutText(image[0], label, rect.Location, FontFace.HersheyComplex, 1.0,
new MCvScalar(0, 0, 255), 2);
int[] masksDim = masks.SizeOfDimension;
using (Mat mask = new Mat(
masksDim[2],
masksDim[3],
DepthType.Cv32F,
1,
//masks.DataPointer +
//(i * masksDim[1] + classId )
//* masksDim[2] * masksDim[3] * masks.ElementSize,
masks.GetDataPointer(i, classId),
masksDim[3] * masks.ElementSize))
using (Mat maskLarge = new Mat())
using (Mat maskLargeInv = new Mat())
using (Mat subRegion = new Mat(image[0], rect))
using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3))
{
CvInvoke.Resize(mask, maskLarge, rect.Size);
//give the mask at least 30% transparency
using (ScalarArray sa = new ScalarArray(0.7))
CvInvoke.Min(sa, maskLarge, maskLarge);
//Create the inverse mask for the original image
using (ScalarArray sa = new ScalarArray(1.0))
CvInvoke.Subtract(sa, maskLarge, maskLargeInv);
//The mask color
largeColor.SetTo(new Emgu.CV.Structure.MCvScalar(255, 0, 0));
if (subRegion.NumberOfChannels == 4)
{
using (Mat bgrSubRegion = new Mat())
{
CvInvoke.CvtColor(subRegion, bgrSubRegion, ColorConversion.Bgra2Bgr);
CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge, maskLargeInv, bgrSubRegion);
CvInvoke.CvtColor(bgrSubRegion, subRegion, ColorConversion.Bgr2Bgra);
}
}
else
{
CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion);
}
}
}
}
}
}
long time = 0;
return(new Tuple <Mat, String, long>(image[0], null, time));
});
t.Start();
var result = await t;
SetImage(t.Result.Item1);
String computeDevice = CvInvoke.UseOpenCL ? "OpenCL: " + Ocl.Device.Default.Name : "CPU";
SetMessage(t.Result.Item2);
};
}
/// <summary>
/// Detect vehicle from the given image
/// </summary>
/// <param name="image">The image</param>
/// <returns>The detected vehicles.</returns>
public Vehicle[] Detect(Mat image)
{
int imgDim = 300;
int vehicleAttrSize = 72;
MCvScalar meanVal = new MCvScalar();
double scale = 1.0;
float vehicleConfidenceThreshold = 0.5f;
float licensePlateConfidenceThreshold = 0.5f;
//MCvScalar meanVal = new MCvScalar(127.5, 127.5, 127.5);
//double scale = 127.5;
Size imageSize = image.Size;
using (Mat inputBlob = DnnInvoke.BlobFromImage(
image,
scale,
new Size(imgDim, imgDim),
meanVal,
false,
false,
DepthType.Cv32F))
_vehicleLicensePlateDetector.SetInput(inputBlob, "Placeholder");
List <Vehicle> vehicles = new List <Vehicle>();
List <LicensePlate> plates = new List <LicensePlate>();
using (Mat detection = _vehicleLicensePlateDetector.Forward("DetectionOutput_"))
{
float[,,,] values = detection.GetData(true) as float[, , , ];
for (int i = 0; i < values.GetLength(2); i++)
{
float imageId = values[0, 0, i, 0];
float label = values[0, 0, i, 1]; //if label == 1, it is a vehicle; if label == 2, it is a license plate
float confident = values[0, 0, i, 2];
float xLeftBottom = values[0, 0, i, 3] * imageSize.Width;
float yLeftBottom = values[0, 0, i, 4] * imageSize.Height;
float xRightTop = values[0, 0, i, 5] * imageSize.Width;
float yRightTop = values[0, 0, i, 6] * imageSize.Height;
RectangleF objectRegion = new RectangleF(
xLeftBottom,
yLeftBottom,
xRightTop - xLeftBottom,
yRightTop - yLeftBottom);
Rectangle region = Rectangle.Round(objectRegion);
if (label == 1 && confident > vehicleConfidenceThreshold)
{ //this is a vehicle
Vehicle v = new Vehicle();
v.Region = region;
#region find out the type and color of the vehicle
using (Mat vehicle = new Mat(image, region))
{
using (Mat vehicleBlob = DnnInvoke.BlobFromImage(
vehicle,
scale,
new Size(vehicleAttrSize, vehicleAttrSize),
meanVal,
false,
false,
DepthType.Cv32F))
{
_vehicleAttrRecognizer.SetInput(vehicleBlob, "input");
using (VectorOfMat vm = new VectorOfMat(2))
{
_vehicleAttrRecognizer.Forward(vm, new string[] { "color", "type" });
using (Mat vehicleColorMat = vm[0])
using (Mat vehicleTypeMat = vm[1])
{
float[] vehicleColorData = vehicleColorMat.GetData(false) as float[];
float maxProbColor = vehicleColorData.Max();
int maxIdxColor = Array.IndexOf(vehicleColorData, maxProbColor);
v.Color = _colorName[maxIdxColor];
float[] vehicleTypeData = vehicleTypeMat.GetData(false) as float[];
float maxProbType = vehicleTypeData.Max();
int maxIdxType = Array.IndexOf(vehicleTypeData, maxProbType);
v.Type = _vehicleType[maxIdxType];
}
}
}
}
#endregion
vehicles.Add(v);
}
if (label == 2 && confident > licensePlateConfidenceThreshold)
{ //this is a license plate
LicensePlate p = new LicensePlate();
p.Region = region;
#region OCR on license plate
using (Mat plate = new Mat(image, region))
{
using (Mat inputBlob = DnnInvoke.BlobFromImage(
plate,
scale,
new Size(94, 24),
meanVal,
false,
false,
DepthType.Cv32F))
using (Mat seqInd = new Mat(
new Size(1, 88),
DepthType.Cv32F,
1))
{
_ocr.SetInput(inputBlob, "data");
if (seqInd.Depth == DepthType.Cv32F)
{
float[] seqIndValues = new float[seqInd.Width * seqInd.Height];
for (int j = 1; j < seqIndValues.Length; j++)
{
seqIndValues[j] = 1.0f;
}
seqIndValues[0] = 0.0f;
seqInd.SetTo(seqIndValues);
}
_ocr.SetInput(seqInd, "seq_ind");
using (Mat output = _ocr.Forward("decode"))
{
float[] plateValue = output.GetData(false) as float[];
StringBuilder licensePlateStringBuilder = new StringBuilder();
foreach (int j in plateValue)
{
if (j >= 0)
{
licensePlateStringBuilder.Append(_plateText[j]);
}
}
p.Text = licensePlateStringBuilder.ToString();
}
}
}
#endregion
plates.Add(p);
}
}
foreach (LicensePlate p in plates)
{
foreach (Vehicle v in vehicles)
{
if (v.ContainsPlate(p))
{
v.LicensePlate = p;
break;
}
}
}
}
return(vehicles.ToArray());
}
/// <summary>
/// Detect vehicle from the given image
/// </summary>
/// <param name="image">The image</param>
/// <returns>The detected vehicles.</returns>
public Vehicle[] Detect(IInputArray image)
{
float vehicleConfidenceThreshold = 0.5f;
float licensePlateConfidenceThreshold = 0.5f;
double scale = 1.0;
MCvScalar meanVal = new MCvScalar();
List <Vehicle> vehicles = new List <Vehicle>();
List <LicensePlate> plates = new List <LicensePlate>();
using (InputArray iaImage = image.GetInputArray())
using (Mat iaImageMat = iaImage.GetMat())
foreach (DetectedObject vehicleOrPlate in _vehicleLicensePlateDetectionModel.Detect(image, 0.0f, 0.0f))
{
Rectangle region = vehicleOrPlate.Region;
if (vehicleOrPlate.ClassId == 1 && vehicleOrPlate.Confident > vehicleConfidenceThreshold)
{
//this is a vehicle
Vehicle v = new Vehicle();
v.Region = region;
#region find out the type and color of the vehicle
using (Mat vehicle = new Mat(iaImageMat, region))
using (VectorOfMat vm = new VectorOfMat(2))
{
_vehicleAttrRecognizerModel.Predict(vehicle, vm);
//_vehicleAttrRecognizer.Forward(vm, new string[] { "color", "type" });
using (Mat vehicleColorMat = vm[0])
using (Mat vehicleTypeMat = vm[1])
{
float[] vehicleColorData = vehicleColorMat.GetData(false) as float[];
float maxProbColor = vehicleColorData.Max();
int maxIdxColor = Array.IndexOf(vehicleColorData, maxProbColor);
v.Color = _colorName[maxIdxColor];
float[] vehicleTypeData = vehicleTypeMat.GetData(false) as float[];
float maxProbType = vehicleTypeData.Max();
int maxIdxType = Array.IndexOf(vehicleTypeData, maxProbType);
v.Type = _vehicleType[maxIdxType];
}
}
#endregion
vehicles.Add(v);
}
else if (vehicleOrPlate.ClassId == 2 && vehicleOrPlate.Confident > licensePlateConfidenceThreshold)
{
//this is a license plate
LicensePlate p = new LicensePlate();
p.Region = region;
#region OCR on license plate
using (Mat plate = new Mat(iaImageMat, region))
{
using (Mat inputBlob = DnnInvoke.BlobFromImage(
plate,
scale,
new Size(94, 24),
meanVal,
false,
false,
DepthType.Cv32F))
{
_ocr.SetInput(inputBlob, "data");
using (Mat output = _ocr.Forward("decode"))
{
float[] plateValue = output.GetData(false) as float[];
StringBuilder licensePlateStringBuilder = new StringBuilder();
foreach (int j in plateValue)
{
if (j >= 0)
{
licensePlateStringBuilder.Append(_plateText[j]);
}
}
p.Text = licensePlateStringBuilder.ToString();
}
}
}
#endregion
plates.Add(p);
}
}
foreach (LicensePlate p in plates)
{
foreach (Vehicle v in vehicles)
{
if (v.ContainsPlate(p))
{
v.LicensePlate = p;
break;
}
}
}
return(vehicles.ToArray());
}
public FaceLandmarkDetectionPage()
: base()
{
var button = this.GetButton();
button.Text = "Perform Face Landmark Detection";
button.Clicked += OnButtonClicked;
OnImagesLoaded += async(sender, image) =>
{
if (image == null || image[0] == null)
{
return;
}
SetMessage("Please wait...");
SetImage(null);
Task <Tuple <IInputArray, long> > t = new Task <Tuple <IInputArray, long> >(
() =>
{
InitFaceDetector();
InitFacemark();
int imgDim = 300;
MCvScalar meanVal = new MCvScalar(104, 177, 123);
Stopwatch watch = Stopwatch.StartNew();
Size imageSize = image[0].Size;
using (Mat inputBlob = DnnInvoke.BlobFromImage(
image[0],
1.0,
new Size(imgDim, imgDim),
meanVal,
false,
false))
_faceDetector.SetInput(inputBlob, "data");
using (Mat detection = _faceDetector.Forward("detection_out"))
{
float confidenceThreshold = 0.5f;
List <Rectangle> faceRegions = new List <Rectangle>();
float[,,,] values = detection.GetData(true) as float[, , , ];
for (int i = 0; i < values.GetLength(2); i++)
{
float confident = values[0, 0, i, 2];
if (confident > confidenceThreshold)
{
float xLeftBottom = values[0, 0, i, 3] * imageSize.Width;
float yLeftBottom = values[0, 0, i, 4] * imageSize.Height;
float xRightTop = values[0, 0, i, 5] * imageSize.Width;
float yRightTop = values[0, 0, i, 6] * imageSize.Height;
RectangleF objectRegion = new RectangleF(
xLeftBottom,
yLeftBottom,
xRightTop - xLeftBottom,
yRightTop - yLeftBottom);
Rectangle faceRegion = Rectangle.Round(objectRegion);
faceRegions.Add(faceRegion);
}
}
using (VectorOfRect vr = new VectorOfRect(faceRegions.ToArray()))
using (VectorOfVectorOfPointF landmarks = new VectorOfVectorOfPointF())
{
_facemark.Fit(image[0], vr, landmarks);
foreach (Rectangle face in faceRegions)
{
CvInvoke.Rectangle(image[0], face, new MCvScalar(0, 255, 0));
}
int len = landmarks.Size;
for (int i = 0; i < landmarks.Size; i++)
{
using (VectorOfPointF vpf = landmarks[i])
FaceInvoke.DrawFacemarks(image[0], vpf, new MCvScalar(255, 0, 0));
}
}
watch.Stop();
return(new Tuple <IInputArray, long>(image[0], watch.ElapsedMilliseconds));
}
});
t.Start();
var result = await t;
SetImage(t.Result.Item1);
String computeDevice = CvInvoke.UseOpenCL ? "OpenCL: " + Ocl.Device.Default.Name : "CPU";
SetMessage(String.Format("Detected in {0} milliseconds.", t.Result.Item2));
};
}
private void timer1_Tick(object sender, EventArgs e)
{
using (Image <Bgr, byte> frame = camera.QueryFrame().ToImage <Bgr, byte>())
{
if (frame != null)
{
Image <Gray, Byte> grayImage = frame.Convert <Gray, byte>();
var StoreEyes = eyes_detect.DetectMultiScale(grayImage);
CvInvoke.Flip(frame, frame, Emgu.CV.CvEnum.FlipType.Horizontal);
Mat blobs = DnnInvoke.BlobFromImage(frame, 1.0, new System.Drawing.Size(detectionSize, detectionSize));
net.SetInput(blobs);
Mat detections = net.Forward();
float[,,,] detectionsArrayInFloats = detections.GetData() as float[, , , ];
for (int i = 0; i < detectionsArrayInFloats.GetLength(2); i++)
{
if (Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 2], CultureInfo.InvariantCulture) > 0.4)
{
float Xstart = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 3],
CultureInfo.InvariantCulture) * detectionSize * xRate;
float Ystart = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 4],
CultureInfo.InvariantCulture) * detectionSize * yRate;
float Xend = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 5],
CultureInfo.InvariantCulture) * detectionSize * xRate;
float Yend = Convert.ToSingle(detectionsArrayInFloats[0, 0, i, 6],
CultureInfo.InvariantCulture) * detectionSize * yRate;
System.Drawing.Rectangle rect = new System.Drawing.Rectangle
{
X = (int)Xstart,
Y = (int)Ystart,
Height = (int)(Yend - Ystart),
Width = (int)(Xend - Xstart)
};
frame.Draw(rect, new Bgr(0, 255, 0), 2);
foreach (var hEye in StoreEyes)
{
//frame.Draw(hEye, new Bgr(0, double.MaxValue, 0), 3);
var avgEyes = StoreEyes?.Average(it => (it.Right + it.Left) / 2) ?? 0;
var turnLeft = (Xstart + Xend) * 0.58;
var turnRight = (Xstart + Xend) * 0.42;
Console.WriteLine($"Xstart in Eyes = {Xstart}");
Console.WriteLine($"Ystart in Eyes = {Ystart}");
Console.WriteLine($"turnLeft = {turnLeft}");
Console.WriteLine($"turnRight = {turnRight}");
Console.WriteLine($"avgEyes = {avgEyes}");
if (avgEyes <= turnLeft && avgEyes >= turnRight)
{
text = "facing straight";
}
else if (avgEyes > turnLeft)
{
text = "turn left";
}
else if (avgEyes < turnRight)
{
text = "turn right";
}
}
CvInvoke.PutText(frame, text, new Point(rect.X - 2, rect.Y - 2), Emgu.CV.CvEnum.FontFace.HersheySimplex, 1.0, new Bgr(Color.Red).MCvScalar);
}
}
imageBox1.Image = frame;
}
}
}
public DetectedObject[] Detect(Mat image, double confThreshold = 0.5)
{
MCvScalar meanVal = new MCvScalar();
Size imageSize = image.Size;
DnnInvoke.BlobFromImage(
image,
_inputBlob,
1.0,
new Size(416, 416),
meanVal,
true,
false,
DepthType.Cv8U);
_yoloDetector.SetInput(_inputBlob, "", 0.00392);
int[] outLayers = _yoloDetector.UnconnectedOutLayers;
String outLayerType = _yoloDetector.GetLayer(outLayers[0]).Type;
String[] outLayerNames = _yoloDetector.UnconnectedOutLayersNames;
using (VectorOfMat outs = new VectorOfMat())
{
List <DetectedObject> detectedObjects = new List <DetectedObject>();
_yoloDetector.Forward(outs, outLayerNames);
if (outLayerType.Equals("Region"))
{
int size = outs.Size;
for (int i = 0; i < size; i++)
{
// Network produces output blob with a shape NxC where N is a number of
// detected objects and C is a number of classes + 4 where the first 4
// numbers are [center_x, center_y, width, height]
using (Mat m = outs[i])
{
int rows = m.Rows;
int cols = m.Cols;
float[,] data = m.GetData(true) as float[, ];
for (int j = 0; j < rows; j++)
{
using (Mat subM = new Mat(m, new Emgu.CV.Structure.Range(j, j + 1), new Emgu.CV.Structure.Range(5, cols)))
{
double minVal = 0, maxVal = 0;
Point minLoc = new Point();
Point maxLoc = new Point();
CvInvoke.MinMaxLoc(subM, ref minVal, ref maxVal, ref minLoc, ref maxLoc);
if (maxVal > confThreshold)
{
int centerX = (int)(data[j, 0] * imageSize.Width);
int centerY = (int)(data[j, 1] * imageSize.Height);
int width = (int)(data[j, 2] * imageSize.Width);
int height = (int)(data[j, 3] * imageSize.Height);
int left = centerX - width / 2;
int top = centerY - height / 2;
Rectangle rect = new Rectangle(left, top, width, height);
DetectedObject obj = new DetectedObject();
obj.ClassId = maxLoc.X;
obj.Confident = maxVal;
obj.Region = rect;
obj.Label = _labels[obj.ClassId];
detectedObjects.Add(obj);
}
}
}
}
}
return(detectedObjects.ToArray());
}
else
{
throw new Exception(String.Format("Unknown output layer type: {0}", outLayerType));
}
}
}
static void Main()
{
//set random color
Random rnd = new Random();
for (int i = 0; i < 21; i++)
{
Colors[i] = new Rgb(rnd.Next(0, 256), rnd.Next(0, 256), rnd.Next(0, 256)).MCvScalar;
}
//get image and set model
//Mat img = CvInvoke.Imread("bali-crop.jpg");
Mat img = CvInvoke.Imread("fish-bike.jpg");
var blob = DnnInvoke.BlobFromImage(img, 1, new Size(512, 515));
var prototxt = "deploy.prototxt";
var model = "VGG_VOC0712Plus_SSD_512x512_ft_iter_160000.caffemodel";
var net = new Net();
var import = Importer.CreateCaffeImporter(prototxt, model);
import.PopulateNet(net);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward("detection_out");
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
//copy result to byte due to egmucv can't access Mat pixel.
byte[] data = new byte[5600];
prob.CopyTo(data);
//draw result
for (int i = 0; i < prob.SizeOfDimemsion[2]; i++)
{
var d = BitConverter.ToSingle(data, i * 28 + 8);
if (d > 0.4)
{
var idx = (int)BitConverter.ToSingle(data, i * 28 + 4);
var w1 = (int)(BitConverter.ToSingle(data, i * 28 + 12) * img.Width);
var h1 = (int)(BitConverter.ToSingle(data, i * 28 + 16) * img.Height);
var w2 = (int)(BitConverter.ToSingle(data, i * 28 + 20) * img.Width);
var h2 = (int)(BitConverter.ToSingle(data, i * 28 + 24) * img.Height);
var label = $"{Labels[idx]} {d * 100:0.00}%";
Console.WriteLine(label);
CvInvoke.Rectangle(img, new Rectangle(w1, h1, w2 - w1, h2 - h1), Colors[idx], 2);
int baseline = 0;
var textSize = CvInvoke.GetTextSize(label, FontFace.HersheyTriplex, 0.5, 1, ref baseline);
var y = h1 - textSize.Height < 0 ? h1 + textSize.Height : h1;
CvInvoke.Rectangle(img, new Rectangle(w1, y - textSize.Height, textSize.Width, textSize.Height), Colors[idx], -1);
CvInvoke.PutText(img, label, new Point(w1, y), FontFace.HersheyTriplex, 0.5, new Bgr(0, 0, 0).MCvScalar);
}
}
//Show the image
CvInvoke.Imshow("image", img);
CvInvoke.WaitKey();
CvInvoke.DestroyAllWindows();
}
/// <summary>
/// Perform detection on the input image and return the results
/// </summary>
/// <param name="m">The input image</param>
/// <param name="matchScoreThreshold">A threshold used to filter boxes by score.</param>
/// <param name="nmsThreshold">A threshold used in non maximum suppression.</param>
/// <returns>The detected objects</returns>
public MaskedObject[] Detect(IInputArray m, float matchScoreThreshold = 0.5f, float nmsThreshold = 0.4f)
{
using (InputArray iaM = m.GetInputArray())
using (Mat blob = DnnInvoke.BlobFromImage(m))
using (VectorOfMat tensors = new VectorOfMat())
{
_maskRcnnDetector.SetInput(blob, "image_tensor");
_maskRcnnDetector.Forward(tensors, new string[] { "detection_out_final", "detection_masks" });
using (Mat boxes = tensors[0])
using (Mat masks = tensors[1])
{
System.Drawing.Size imgSize = iaM.GetSize();
float[,,,] boxesData = boxes.GetData(true) as float[, , , ];
int numDetections = boxesData.GetLength(2);
List <int> classIds = new List <int>();
List <Rectangle> regions = new List <Rectangle>();
List <float> scores = new List <float>();
for (int i = 0; i < numDetections; i++)
{
int classId = (int)boxesData[0, 0, i, 1];
if (_objectsOfInterest == null || _objectsOfInterest.Contains(_labels[classId]))
{
float score = boxesData[0, 0, i, 2];
Rectangle rect = DetectedObject.GetRectangle(
boxesData[0, 0, i, 3],
boxesData[0, 0, i, 4],
boxesData[0, 0, i, 5],
boxesData[0, 0, i, 6],
imgSize.Width,
imgSize.Height);
rect.Intersect(new Rectangle(Point.Empty, imgSize));
regions.Add(rect);
scores.Add(score);
classIds.Add(classId);
}
}
int[] validIdx = DnnInvoke.NMSBoxes(regions.ToArray(), scores.ToArray(), matchScoreThreshold, nmsThreshold);
List <MaskedObject> maskedObjects = new List <MaskedObject>();
for (int i = 0; i < validIdx.Length; i++)
{
int idx = validIdx[i];
int classId = classIds[idx];
Rectangle rect = regions[idx];
float score = scores[idx];
int[] masksDim = masks.SizeOfDimension;
using (Mat mask = new Mat(
masksDim[2],
masksDim[3],
DepthType.Cv32F,
1,
masks.GetDataPointer(i, classId),
masksDim[3] * masks.ElementSize))
{
MaskedObject mo = new MaskedObject(classId, _labels[classId], score, rect, mask);
maskedObjects.Add(mo);
}
}
return(maskedObjects.ToArray());
}
}
}
private void PoseEstimationBody_25_Click(object sender, EventArgs e)
{
try
{
if (!IMGDict.ContainsKey("input"))
{
throw new Exception("Please read in image first.");
}
// for openopse
int inWidth = 368;
int inHeight = 368;
float threshold = 0.1f;
int nPoints = 25;
var BODY_PARTS = new Dictionary <string, int>()
{
{ "Nose", 0 },
{ "Neck", 1 },
{ "RShoulder", 2 },
{ "RElbow", 3 },
{ "RWrist", 4 },
{ "LShoulder", 5 },
{ "LElbow", 6 },
{ "LWrist", 7 },
{ "MidHip", 8 },
{ "RHip", 9 },
{ "RKnee", 10 },
{ "RAnkle", 11 },
{ "LHip", 12 },
{ "LKnee", 13 },
{ "LAnkle", 14 },
{ "REye", 15 },
{ "LEye", 16 },
{ "REar", 17 },
{ "LEar", 18 },
{ "LBigToe", 19 },
{ "LSmallToe", 20 },
{ "LHeel", 21 },
{ "RBigToe", 22 },
{ "RSmallToe", 23 },
{ "RHeel", 24 },
{ "Background", 25 }
};
int[,] point_pairs = new int[, ] {
{ 1, 0 }, { 1, 2 }, { 1, 5 },
{ 2, 3 }, { 3, 4 }, { 5, 6 },
{ 6, 7 }, { 0, 15 }, { 15, 17 },
{ 0, 16 }, { 16, 18 }, { 1, 8 },
{ 8, 9 }, { 9, 10 }, { 10, 11 },
{ 11, 22 }, { 22, 23 }, { 11, 24 },
{ 8, 12 }, { 12, 13 }, { 13, 14 },
{ 14, 19 }, { 19, 20 }, { 14, 21 }
};
// Load the caffe Model
string prototxt = @"F:\openpose\models\pose\body_25\pose_deploy.prototxt";
string modelPath = @"F:\openpose\models\pose\body_25\pose_iter_584000.caffemodel";
var net = DnnInvoke.ReadNetFromCaffe(prototxt, modelPath);
var img = IMGDict["input"].Clone();
var imgHeight = img.Height;
var imgWidth = img.Width;
var blob = DnnInvoke.BlobFromImage(img, 1.0 / 255.0, new Size(inWidth, inHeight), new MCvScalar(0, 0, 0));
net.SetInput(blob);
net.SetPreferableBackend(Emgu.CV.Dnn.Backend.OpenCV);
var output = net.Forward();
var H = output.SizeOfDimension[2];
var W = output.SizeOfDimension[3];
var HeatMap = output.GetData();
var points = new List <Point>();
for (int i = 0; i < nPoints; i++)
{
Matrix <float> matrix = new Matrix <float>(H, W);
for (int row = 0; row < H; row++)
{
for (int col = 0; col < W; col++)
{
matrix[row, col] = (float)HeatMap.GetValue(0, i, row, col);
}
}
double minVal = 0, maxVal = 0;
Point minLoc = default, maxLoc = default;
public DnnPage()
: base()
{
var button = this.GetButton();
button.Text = "Perform Mask-rcnn Detection";
button.Clicked += OnButtonClicked;
OnImagesLoaded += async(sender, image) =>
{
if (image == null || image[0] == null)
{
return;
}
SetMessage("Please wait...");
SetImage(null);
Task <Tuple <Mat, String, long> > t = new Task <Tuple <Mat, String, long> >(
() =>
{
InitDetector();
String msg = String.Empty;
using (Mat blob = DnnInvoke.BlobFromImage(image[0]))
using (VectorOfMat tensors = new VectorOfMat())
{
_maskRcnnDetector.SetInput(blob, "image_tensor");
Stopwatch watch = Stopwatch.StartNew();
_maskRcnnDetector.Forward(tensors, new string[] { "detection_out_final", "detection_masks" });
watch.Stop();
msg = String.Format("Mask RCNN inception completed in {0} milliseconds.",
watch.ElapsedMilliseconds);
using (Mat boxes = tensors[0])
using (Mat masks = tensors[1])
{
System.Drawing.Size imgSize = image[0].Size;
float[,,,] boxesData = boxes.GetData(true) as float[, , , ];
int numDetections = boxesData.GetLength(2);
for (int i = 0; i < numDetections; i++)
{
float score = boxesData[0, 0, i, 2];
if (score > 0.5)
{
int classId = (int)boxesData[0, 0, i, 1];
String label = _labels[classId];
MCvScalar color = _colors[classId];
float left = boxesData[0, 0, i, 3] * imgSize.Width;
float top = boxesData[0, 0, i, 4] * imgSize.Height;
float right = boxesData[0, 0, i, 5] * imgSize.Width;
float bottom = boxesData[0, 0, i, 6] * imgSize.Height;
RectangleF rectF = new RectangleF(left, top, right - left, bottom - top);
Rectangle rect = Rectangle.Round(rectF);
rect.Intersect(new Rectangle(Point.Empty, imgSize));
CvInvoke.Rectangle(image[0], rect, new MCvScalar(0, 0, 0, 0), 1);
CvInvoke.PutText(image[0], label, rect.Location, FontFace.HersheyComplex, 1.0,
new MCvScalar(0, 0, 255), 2);
int[] masksDim = masks.SizeOfDimension;
using (Mat mask = new Mat(
masksDim[2],
masksDim[3],
DepthType.Cv32F,
1,
masks.GetDataPointer(i, classId),
masksDim[3] * masks.ElementSize))
using (Mat maskLarge = new Mat())
using (Mat maskLargeInv = new Mat())
using (Mat subRegion = new Mat(image[0], rect))
using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U, 3))
{
CvInvoke.Resize(mask, maskLarge, rect.Size);
//give the mask at least 30% transparency
using (ScalarArray sa = new ScalarArray(0.7))
CvInvoke.Min(sa, maskLarge, maskLarge);
//Create the inverse mask for the original image
using (ScalarArray sa = new ScalarArray(1.0))
CvInvoke.Subtract(sa, maskLarge, maskLargeInv);
//The mask color
largeColor.SetTo(color);
if (subRegion.NumberOfChannels == 4)
{
using (Mat bgrSubRegion = new Mat())
{
CvInvoke.CvtColor(subRegion, bgrSubRegion, ColorConversion.Bgra2Bgr);
CvInvoke.BlendLinear(largeColor, bgrSubRegion, maskLarge, maskLargeInv, bgrSubRegion);
CvInvoke.CvtColor(bgrSubRegion, subRegion, ColorConversion.Bgr2Bgra);
}
}
else
{
CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv, subRegion);
}
}
}
}
}
}
long time = 0;
return(new Tuple <Mat, String, long>(image[0], msg, time));
});
t.Start();
var result = await t;
SetImage(t.Result.Item1);
//String computeDevice = CvInvoke.UseOpenCL ? "OpenCL: " + Ocl.Device.Default.Name : "CPU";
SetMessage(t.Result.Item2);
};
}
private void DetectAndRender(Mat image)
{
int imgDim = 300;
MCvScalar meanVal = new MCvScalar(104, 177, 123);
Size imageSize = image.Size;
using (Mat inputBlob = DnnInvoke.BlobFromImage(
image,
1.0,
new Size(imgDim, imgDim),
meanVal,
false,
false))
_faceDetector.SetInput(inputBlob, "data");
using (Mat detection = _faceDetector.Forward("detection_out"))
{
float confidenceThreshold = 0.5f;
List <Rectangle> faceRegions = new List <Rectangle>();
float[,,,] values = detection.GetData(true) as float[, , , ];
for (int i = 0; i < values.GetLength(2); i++)
{
float confident = values[0, 0, i, 2];
if (confident > confidenceThreshold)
{
float xLeftBottom = values[0, 0, i, 3] * imageSize.Width;
float yLeftBottom = values[0, 0, i, 4] * imageSize.Height;
float xRightTop = values[0, 0, i, 5] * imageSize.Width;
float yRightTop = values[0, 0, i, 6] * imageSize.Height;
RectangleF objectRegion = new RectangleF(
xLeftBottom,
yLeftBottom,
xRightTop - xLeftBottom,
yRightTop - yLeftBottom);
Rectangle faceRegion = Rectangle.Round(objectRegion);
faceRegions.Add(faceRegion);
}
}
using (VectorOfRect vr = new VectorOfRect(faceRegions.ToArray()))
using (VectorOfVectorOfPointF landmarks = new VectorOfVectorOfPointF())
{
_facemark.Fit(image, vr, landmarks);
foreach (Rectangle face in faceRegions)
{
CvInvoke.Rectangle(image, face, new MCvScalar(0, 255, 0));
}
int len = landmarks.Size;
for (int i = 0; i < landmarks.Size; i++)
{
using (VectorOfPointF vpf = landmarks[i])
FaceInvoke.DrawFacemarks(image, vpf, new MCvScalar(255, 0, 0));
}
}
}
}
