public override void RunTest()
{
const string protoTxt = @"Data\Text\bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"Data\Text\synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
Console.Write("Downloading Caffe Model...");
PrepareModel(caffeModel);
Console.WriteLine(" Done");
using var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
using var img = new Mat(@"Data\Image\space_shuttle.jpg");
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
// Convert Mat to batch of images
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
net.SetInput(inputBlob, "data");
using var prob = net.Forward("prob");
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability: {0:P2}", classProb);
Console.WriteLine("Press any key to exit");
Console.Read();
}
public void LoadYoloV2Model()
{
RunGC();
const string cfgFile = @"_data/model/yolov2.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2.cfg";
const string darknetModel = "_data/model/yolov2.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov2.weights";
testOutputHelper.WriteLine("Downloading YoloV2 Model...");
PrepareFile(new Uri(cfgFileUrl), cfgFile);
PrepareFile(new Uri(darknetModelUrl), darknetModel);
testOutputHelper.WriteLine("Done");
RunGC();
using var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel);
Assert.NotNull(net);
Assert.False(net !.Empty());
// Convert Mat to batch of images
using var img = Image(@"space_shuttle.jpg");
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using var detectionMat = net.Forward("detection_out");
// TODO
GC.KeepAlive(detectionMat);
}
public void LoadCaffeModel(String source)
{
Mat img = Cv2.ImRead(source);
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
{
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
// Assert.Equal(1, net.GetLayerId(net.GetLayerNames()[0]));
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1.0, new Size(300, 300), new Scalar(104, 117, 123), false, false))
{
net.SetInput(inputBlob, "data");
using (var detection = net.Forward("detection_out"))
{
// find the best class
Console.WriteLine(detection);
Console.WriteLine(detection.Size(2));
GetMaxClass(detection, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} ", classId);
Console.WriteLine("Probability: {0:P2}", classProb);
// Pause();
//Assert.Equal(812, classId);
}
}
}
}
public DnnDetectedObject[] ClassifyObjects(Mat image, Rect boxToAnalyze)
{
if (image == null)
{
throw new ArgumentNullException(nameof(image));
}
var blob = CvDnn.BlobFromImage(image, 1.0 / 255, new Size(320, 320), new Scalar(), crop: false);
nnet.SetInput(blob);
//create mats for output layer
Mat[] outs = Enumerable.Repeat(false, _outNames.Length).Select(_ => new Mat()).ToArray();
//forward model
nnet.Forward(outs, _outNames);
const float threshold = 0.5f; //for confidence
const float nmsThreshold = 0.3f; //threshold for nms
var detections = ExtractYolo2Results(outs, image, threshold, nmsThreshold);
blob.Dispose();
foreach (var output in outs)
{
output.Dispose();
}
return(detections);
}
public Rectangle[] Detect(Mat mat, byte[] buffer)
{
var frameWidth = mat.Width;
var frameHeight = mat.Height;
using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(104, 117, 123), false, false);
Net.SetInput(blob);
using var detections = Net.Forward();
using var detectionMat = new Mat(detections.Size(2), detections.Size(3), MatType.CV_32F, detections.Ptr(0));
var rectangles = new List <Rectangle>();
for (var i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
var left = (int)(detectionMat.At <float>(i, 3) * frameWidth);
var top = (int)(detectionMat.At <float>(i, 4) * frameHeight);
var right = (int)(detectionMat.At <float>(i, 5) * frameWidth);
var bottom = (int)(detectionMat.At <float>(i, 6) * frameHeight);
rectangles.Add(new Rectangle(left, top, right - left, bottom - top));
}
}
return(rectangles.ToArray());
}
public Rectangle[] Detect(Mat mat, byte[] buffer)
{
var frameWidth = mat.Width;
var frameHeight = mat.Height;
//using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(104, 117, 123), false, false);
using var blob = CvDnn.BlobFromImage(mat, 1.0 / Mean, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(Mean), true, false);
Net.SetInput(blob);
using var detections = Net.Forward();
using var detectionMat = new Mat(detections.Size(2), detections.Size(3), MatType.CV_32F, detections.Ptr(0));
var bestConfidence = 0.0;
Rectangle?candidate = null;
for (var i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
if ((confidence > 0.5) && (confidence > bestConfidence))
{
var left = (int)(detectionMat.At <float>(i, 3) * frameWidth);
var top = (int)(detectionMat.At <float>(i, 4) * frameHeight);
var right = (int)(detectionMat.At <float>(i, 5) * frameWidth);
var bottom = (int)(detectionMat.At <float>(i, 6) * frameHeight);
candidate = new Rectangle(left, top, right - left, bottom - top);
}
}
if (candidate != null)
{
return(new Rectangle[] { candidate.Value });
}
else
{
return(Array.Empty <Rectangle>());
}
}
public async Task LoadYoloV2Model()
{
const string cfgFile = @"yolov2.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2.cfg";
const string darknetModel = "yolov2.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov2.weights";
Console.Write("Downloading YoloV2 Model...");
await PrepareFile(cfgFileUrl, cfgFile);
await PrepareFile(darknetModelUrl, darknetModel);
Console.WriteLine(" Done");
using (var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel))
using (var img = Image(@"space_shuttle.jpg"))
{
Assert.False(net.Empty());
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using (var detectionMat = net.Forward("detection_out"))
{
// TODO
GC.KeepAlive(detectionMat);
}
}
}
}
/// <summary>
/// Begin detect pass the blob to the dnn network
/// </summary>
/// <param name="img">Image to detect</param>
/// <param name="minProbability">Min probability to get label</param>
/// <param name="labelsFilters">Label to filters</param>
/// <returns>Net Result</returns>
protected override NetResult[] BeginDetect(Bitmap img, float minProbability = 0.3F, string[] labelsFilters = null)
{
using (Mat mat = OpenCvSharp.Extensions.BitmapConverter.ToMat(img))
{
//Create a blob
var blob = CvDnn.BlobFromImage(mat, Scale, size: new OpenCvSharp.Size(256, 256), mean: new Scalar(78.4263377603, 87.7689143744, 114.895847746), swapRB: false);
//Set the input for the layer "data"
network.SetInput(blob, "data");
//Get result of the layer prob
var prop = network.Forward("prob");
//Get the maxLoc and max for probability and the label index
prop.MinMaxLoc(out double min, out double max, out OpenCvSharp.Point minLoc, out OpenCvSharp.Point maxLoc);
return(new NetResult[]
{
new NetResult()
{
Label = Labels[maxLoc.X],
Probability = max
}
});
}
}
public void LoadCaffeModel()
{
const string protoTxt = @"_data/text/bvlc_googlenet.prototxt";
const string caffeModelUrl = "https://drive.google.com/uc?id=1RUsoiLiXrKBQu9ibwsMqR3n_UkhnZLRR"; //"http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel";
const string caffeModel = "_data/model/bvlc_googlenet.caffemodel";
const string synsetWords = @"_data/text/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
testOutputHelper.WriteLine("Downloading Caffe Model...");
PrepareModel(caffeModelUrl, caffeModel);
testOutputHelper.WriteLine("Done");
using var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
//Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
var layerName = net.GetLayerNames()[0];
Assert.NotNull(layerName);
Assert.Equal(1, net.GetLayerId(layerName !));
// Convert Mat to batch of images
using var img = Image(@"space_shuttle.jpg");
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
net.SetInput(inputBlob, "data");
using var prob = net.Forward("prob");
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
testOutputHelper.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
testOutputHelper.WriteLine("Probability: {0:P2}", classProb);
Pause();
Assert.Equal(812, classId);
}
public override void RunTest()
{
// Read sample image
using var frame = Cv2.ImRead(image);
int frameHeight = frame.Rows;
int frameWidth = frame.Cols;
using var faceNet = CvDnn.ReadNetFromCaffe(configFile, faceModel);
using var blob = CvDnn.BlobFromImage(frame, 1.0, new Size(300, 300), new Scalar(104, 117, 123), false, false);
faceNet.SetInput(blob, "data");
using var detection = faceNet.Forward("detection_out");
using var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F,
detection.Ptr(0));
for (int i = 0; i < detectionMat.Rows; i++)
{
float confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
int x1 = (int)(detectionMat.At <float>(i, 3) * frameWidth);
int y1 = (int)(detectionMat.At <float>(i, 4) * frameHeight);
int x2 = (int)(detectionMat.At <float>(i, 5) * frameWidth);
int y2 = (int)(detectionMat.At <float>(i, 6) * frameHeight);
Cv2.Rectangle(frame, new Point(x1, y1), new Point(x2, y2), new Scalar(0, 255, 0), 2, LineTypes.Link4);
}
}
Window.ShowImages(frame);
}
private string OpenCVDeepLearningDetector(string path)
{
// uses emugu library
//https://medium.com/@vinuvish/face-detection-with-opencv-and-deep-learning-90bff9028fa8
string prototextPath = @"./Dnn/deploy.prototxt";
string caffeModelPath = @"./Dnn/res10_300x300_ssd_iter_140000.caffemodel";
// load the model;
var net = CvDnn.ReadNetFromCaffe(prototxt: prototextPath, caffeModel: caffeModelPath);
// get the image
OpenCvSharp.Mat image = Cv2.ImRead(path);
// get the original image size
OpenCvSharp.Size imageSize = image.Size();
// the dnn detector works on a 300x300 image;
// now resize the image for the Dnn dector;
OpenCvSharp.Size size = new OpenCvSharp.Size(299, 299);
image = image.Resize(size);
// set the scalar property to RGB colors, don't know what these values represent.
OpenCvSharp.Scalar mcvScalar = new OpenCvSharp.Scalar(104.0, 177.0, 123.0);
var blob = CvDnn.BlobFromImage(image: image, scaleFactor: 1, size: size, mean: mcvScalar, swapRB: true);
net.SetInput(blob);
OpenCvSharp.Mat detections = net.Forward();
// convert the detected values to a faces object that we can use to
// draw rectangles.
List <ConfidenceRect> Faces = new List <ConfidenceRect>();
//var rows = detections.SizeOfDimension[2];
//Array ans = detections.GetData();
//for (int n = 0; n < rows; n++)
//{
// object confidence = ans.GetValue(0, 0, n, 2);
// object x1 = ans.GetValue(0, 0, n, 3);
// object y1 = ans.GetValue(0, 0, n, 4);
// object x2 = ans.GetValue(0, 0, n, 5);
// object y2 = ans.GetValue(0, 0, n, 6);
// ConfidenceRect cr = new ConfidenceRect(confidence, x1, y1, x2, y2, imageSize);
// if (cr.Confidence > 0)
// {
// Debug.WriteLine($"Confidence {cr.Confidence}");
// }
// if (cr.Confidence > Confidence)
// {
// Faces.Add(cr);
// }
//}
//// convert to a writeableBitmap
//WriteableBitmap writeableBitmap = new WriteableBitmap(ImageSource);
//ImageSource = ConvertWriteableBitmapToBitmapImage(writeableBitmap);
//OnPropertyChanged("ImageSource");
//DrawDnnOnImage?.Invoke(Faces, imageSize);
//return Faces.Count.ToString();
return(null);
}
static void Main(string[] args)
{
const string prototext = @"..\..\..\..\data\bvlc_googlenet.prototxt";
const string caffeModel = @"..\..\..\..\data\bvlc_googlenet.caffemodel";
const string synsetWords = @"..\..\..\..\data\synset_words.txt";
string[] classNames = File.ReadAllLines(synsetWords).Select(l => l.Split(' ').Last()).ToArray();
//Use stopwatch object fro timing of the operation
Stopwatch sw = new Stopwatch();
string imgPath = @"D:\DeepLearningOpenCV\images\DogBycleCar.jpg";
using (var net = CvDnn.ReadNetFromCaffe(prototext, caffeModel))
using (var img = Cv2.ImRead(imgPath))
{
//Just out of curiosity, I wanted to get the Layer names of the NN Construct
// by calling GetLayerNames method of the Net object
string[] layerNames = net.GetLayerNames();
Console.WriteLine("Layer names : {0}", string.Join(", ", layerNames));
Console.WriteLine();
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123), swapRB: true, crop: false))
{
sw.Start();
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
sw.Stop();
Console.WriteLine($"Cost of calculating prob {sw.ElapsedMilliseconds} ms");
int cols = prob.Cols;
int rows = prob.Rows;
Console.WriteLine("Cols: " + cols + ", Rows:" + rows);
// GetMaxProClass(prob, out int classId, out double classProb);
Cv2.MinMaxLoc(prob, out _, out double classProb, out _, out Point classNumberPoint);
int classId = classNumberPoint.X;
Console.WriteLine("Best class: #{0}, '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability:{0:P2}", classProb);
string txt = "Label: " + classNames[classId] + ", % " + (100 * classProb).ToString("0.####");
Cv2.PutText(img, txt, new Point(5, 25), HersheyFonts.HersheySimplex, 0.7, new Scalar(0, 0, 255), 2);
//Cv2.ImWrite("classification.jpg", img);
Cv2.ImShow("image", img);
}
}
}
Cv2.WaitKey();
Cv2.DestroyAllWindows();
// Console.Write("Downloading Caffe Model...");
Console.WriteLine("Press any key to exit");
Console.Read();
}
protected override Mat GetFaces(Mat frame, out FacesList facesList)
{
// debug - replace video stream with still image for testing
//var file = "C:\\Users\\Lamby\\Documents\\Visual Studio 2017\\Projects\\TrackingCamera\\TrackingCamera\\DetectorClasses\\ModelDetectorVGG_VOC0712Plus\\bali-crop.jpg";
string file = "C:\\Users\\Lamby\\Desktop\\fd-acc-result3-e1539872783684.jpg";
frame = Cv2.ImRead(file);
Mat imageBlob = CvDnn.BlobFromImage(frame, 1.0, new Size(300, 300),
new Scalar(104.0, 177.0, 123.0), false, false);
this.Detector.SetInput(imageBlob, "data");
Mat detections = this.Detector.Forward("detection_out");
//reshape from [1,1,200,7] to [200,7]
Mat detectionMat = detections.Reshape(1, detections.Size(2));
// debug
//GetFaceBestConfidence(detections, out int faceId, out double faceProbability);
if (detectionMat.Rows <= 0) //
{
facesList = new FacesList();
return(null);
}
else
{
facesList = new FacesList();
Scalar rgbColour = new Scalar(0, 255, 255);
for (int i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
if (confidence > this.MinConfidence)
{
int X1 = (int)(detectionMat.At <float>(i, 3) * frame.Width); //detectionMat.At<int> returns 0 with this floating point caffe model?
int Y1 = (int)(detectionMat.At <float>(i, 4) * frame.Height);
int X2 = (int)(detectionMat.At <float>(i, 5) * frame.Width);
int Y2 = (int)(detectionMat.At <float>(i, 6) * frame.Height);
frame.Rectangle(new Point(X1, Y1), new Point(X2, Y2), rgbColour, 2, OpenCvSharp.LineTypes.Link4);
string faceText = String.Format("{0:P2}", confidence);
Cv2.PutText(frame, faceText, new Point(X1, Y2 + 9), HersheyFonts.HersheyComplex, 0.3, rgbColour);
var faceMat = frame[new Rect(X1, Y1, X2 - X1, Y2 - Y1)];
facesList.Add(new Face(faceMat, new Point(X1, Y1), new Point(X2, Y2), confidence));
// Debug
//Cv2.ImShow("Detected Face", faceMat);
//Cv2.WaitKey(1);
}
}
return(frame);
}
}
public void DetectAllText(string fileName)
{
const int InputWidth = 320;
const int InputHeight = 320;
const float ConfThreshold = 0.5f;
const float NmsThreshold = 0.4f;
// Load network.
using (Net net = CvDnn.ReadNet(Path.GetFullPath(LocalModelPath)))
using (Mat img = new Mat(fileName))
// Prepare input image
using (var blob = CvDnn.BlobFromImage(img, 1.0, new Size(InputWidth, InputHeight), new Scalar(123.68, 116.78, 103.94), true, false))
{
// Forward Pass
// Now that we have prepared the input, we will pass it through the network. There are two outputs of the network.
// One specifies the geometry of the Text-box and the other specifies the confidence score of the detected box.
// These are given by the layers :
// feature_fusion/concat_3
// feature_fusion/Conv_7/Sigmoid
var outputBlobNames = new string[] { "feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3" };
var outputBlobs = outputBlobNames.Select(_ => new Mat()).ToArray();
net.SetInput(blob);
net.Forward(outputBlobs, outputBlobNames);
Mat scores = outputBlobs[0];
Mat geometry = outputBlobs[1];
// Decode predicted bounding boxes (decode the positions of the text boxes along with their orientation)
Decode(scores, geometry, ConfThreshold, out var boxes, out var confidences);
// Apply non-maximum suppression procedure for filtering out the false positives and get the final predictions
CvDnn.NMSBoxes(boxes, confidences, ConfThreshold, NmsThreshold, out var indices);
// Render detections.
Point2f ratio = new Point2f((float)img.Cols / InputWidth, (float)img.Rows / InputHeight);
for (var i = 0; i < indices.Length; ++i)
{
RotatedRect box = boxes[indices[i]];
Point2f[] vertices = box.Points();
for (int j = 0; j < 4; ++j)
{
vertices[j].X *= ratio.X;
vertices[j].Y *= ratio.Y;
}
for (int j = 0; j < 4; ++j)
{
Cv2.Line(img, (int)vertices[j].X, (int)vertices[j].Y, (int)vertices[(j + 1) % 4].X, (int)vertices[(j + 1) % 4].Y, new Scalar(0, 255, 0), 3);
}
}
ShowImagesWhenDebugMode(img);
}
}
public void LoadYoloV3Model()
{
RunGC();
const string cfgFile = @"_data/model/yolov3.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg";
const string darknetModel = "_data/model/yolov3.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov3.weights";
testOutputHelper.WriteLine("Downloading YoloV3 Model...");
PrepareFile(new Uri(cfgFileUrl), cfgFile);
PrepareFile(new Uri(darknetModelUrl), darknetModel);
testOutputHelper.WriteLine("Done");
RunGC();
using (var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel))
using (var img = Image(@"space_shuttle.jpg"))
{
Assert.NotNull(net);
Assert.False(net !.Empty());
var outNames = net.GetUnconnectedOutLayersNames();
Assert.NotEmpty(outNames);
Assert.DoesNotContain(outNames, elem => elem == null);
testOutputHelper.WriteLine("UnconnectedOutLayersNames: {0}", string.Join(",", outNames));
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using (var detection82 = net.Forward("yolo_82"))
using (var detection94 = net.Forward("yolo_94"))
using (var detection106 = net.Forward("yolo_106"))
{
// TODO
Assert.False(detection82.Empty());
Assert.False(detection94.Empty());
Assert.False(detection106.Empty());
}
Mat[] outs = outNames.Select(_ => new Mat()).ToArray();
net.Forward(outs, outNames !);
foreach (var m in outs)
{
Assert.False(m.Empty());
m.Dispose();
}
}
}
}
static void Main()
{
var file = "bali.jpg";
var prototxt = "deploy.prototxt";
var model = "VGG_VOC0712Plus_SSD_512x512_ft_iter_160000.caffemodel";
var colors = Enumerable.Repeat(false, 21).Select(x => Scalar.RandomColor()).ToArray();
//get image
var org = Cv2.ImRead(file);
var blob = CvDnn.BlobFromImage(org, 1, new Size(512, 512));
//setup model
var net = CvDnn.ReadNetFromCaffe(prototxt, model);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward("detection_out");
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
//reshape from [1,1,200,7] to [200,7]
var p = prob.Reshape(1, prob.Size(2));
for (int i = 0; i < prob.Size(2); i++)
{
var confidence = p.At <float>(i, 2);
if (confidence > 0.4)
{
//get value what we need
var idx = (int)p.At <float>(i, 1);
var w1 = (int)(org.Width * p.At <float>(i, 3));
var h1 = (int)(org.Width * p.At <float>(i, 4));
var w2 = (int)(org.Width * p.At <float>(i, 5));
var h2 = (int)(org.Width * p.At <float>(i, 6));
var label = $"{Labels[idx]} {confidence * 100:0.00}%";
Console.WriteLine($"{label}");
//draw result
Cv2.Rectangle(org, new Rect(w1, h1, w2 - w1, h2 - h1), colors[idx], 2);
var textSize = Cv2.GetTextSize(label, HersheyFonts.HersheyTriplex, 0.5, 1, out var baseline);
Cv2.Rectangle(org, new Rect(new Point(w1, h1 - textSize.Height),
new Size(textSize.Width, textSize.Height + baseline)), colors[idx], Cv2.FILLED);
Cv2.PutText(org, label, new Point(w1, h1), HersheyFonts.HersheyTriplex, 0.5, Scalar.Black);
}
}
using (new Window("image", org))
{
Cv2.WaitKey();
}
}
private void bWorker_DoWork(object sender, DoWorkEventArgs e)
{
//Loading YOLO config
Darknet darknet = new Darknet(YoloCfgFile, YoloWeightsFile);
var classNames = File.ReadAllLines(YoloNamesFile);
//Capturing frames
for (; ;)
{
Mat originalBackground = this.background.Clone();
Mat frame;
frame = cam.RetrieveMat();
if (frame.Empty())
{
Cv2.WaitKey();
return;
}
Mat inputBlob = CvDnn.BlobFromImage(frame, 1 / 255d, new OpenCvSharp.Size(544, 544), new Scalar(), true, false);
var results = darknet.Detect(frame, 0.5f);
foreach (var result in results)
{
var p1 = new OpenCvSharp.Point(result.x, result.y);
var p2 = new OpenCvSharp.Point(result.x + result.w, result.y + result.h);
if (CheckContoursAmount(contours, binary, p1, p2))
{
//Formatting labels
var label = $"{classNames[result.obj_id]}";
var textSize = Cv2.GetTextSize(label,
HersheyFonts.HersheyTriplex,
0.5,
1,
out var baseline);
Cv2.Rectangle(originalBackground, p1, p2, Colors[result.obj_id], -1);
Cv2.PutText(
originalBackground,
label,
new OpenCvSharp.Point((int)(result.x + (result.w / 2.0) - (textSize.Width / 2.0)), (int)(result.y + (result.h / 2.0))),
HersheyFonts.HersheyTriplex,
0.5,
Scalar.Black);
}
}
bWorker.ReportProgress(0, originalBackground);
}
}
static void Main()
{
//model can get from http://www.robots.ox.ac.uk/~vgg/software/vgg_face/
var file = "jeremy-clarkson-v2.jpg";
//var file = "john-cena.jpg";
var prototxt = "VGG_FACE_deploy.prototxt";
var model = "VGG_FACE.caffemodel";
var labeltxt = "names.txt";
//read all names
var labels = ReadLabels(labeltxt);
var org = Cv2.ImRead(file);
var blob = CvDnn.BlobFromImage(org, 1, new Size(224, 224));
var net = CvDnn.ReadNetFromCaffe(prototxt, model);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward("prob");
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
//convert result to list
var probList = new Dictionary <int, float>();
for (int i = 0; i < prob.Width; i++)
{
probList.Add(i, prob.At <float>(0, i));
}
//get top 3
var top3 = probList.OrderByDescending(x => x.Value).Take(3).ToList();
foreach (var result in top3)
{
Console.WriteLine($"{labels[result.Key]}:{result.Value*100:0.00}%");
}
//draw result
org.PutText($"{labels[top3.First().Key]}:{top3.First().Value * 100:0.00}%",
new Point(0, 25), HersheyFonts.HersheyTriplex, 1, Scalar.OrangeRed);
using (new Window("image", org))
{
Cv2.WaitKey();
}
}
public string Run(Mat image)
{
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(image, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
return(String.Format("Best class: #{0} '{1}' Probability: {2:P2}", classId, classNames[classId], classProb));
}
}
}
public (Mat OutputImg, List <string> ClassNames, List <Point> Centers, List <float> Confidences, List <Rect2d> Boxes) Run(Mat inputImg)
{
if (_status == Status.NotInitialized)
{
throw new ApplicationException("Detector is not Initialized yet!");
}
var blob = CvDnn.BlobFromImage(inputImg, 1.0 / 255, BlobSize, new Scalar(), true, false);
_net.SetInput(blob);
var outNames = _net.GetUnconnectedOutLayersNames();
var outs = outNames.Select(_ => new Mat()).ToArray();
_net.Forward(outs, outNames);
return(GetResult(outs, inputImg, _threshold, _nmsThreshold));
}
public List <BoundingBox> Inference(Mat x)
{
if (x.Empty())
{
Console.WriteLine("empty x");
return(null);
}
Size rawSize = x.Size();
x = CvDnn.BlobFromImage(x, scaleFactor: 1 / 255.0f, size: inputSize, swapRB: false, crop: false);
net.SetInput(x);
Mat res = net.Forward();
if (res.Empty())
{
Console.WriteLine("empty result");
return(null);
}
List <BoundingBox> boxes = new List <BoundingBox>();
for (int i = 0; i < classCount; ++i)
{
Mat classResult = new Mat(outputHeight, outputWidth, MatType.CV_8UC1);
for (int j = 0; j < outputHeight; ++j)
{
for (int k = 0; k < outputWidth; ++k)
{
float val = res.At <float>(0, i, j, k);
classResult.At <uint>(j, k) = val > 0.5 ? (uint)255 : 0;
}
}
Cv2.Resize(classResult, classResult, rawSize);
Point[][] contours = Cv2.FindContoursAsArray(classResult, RetrievalModes.External, ContourApproximationModes.ApproxSimple);
foreach (Point[] contour in contours)
{
Rect rect = Cv2.BoundingRect(contour);
int x1 = rect.TopLeft.X - boxPaddingVal;
int y1 = rect.TopLeft.Y - boxPaddingVal;
int x2 = rect.BottomRight.X + boxPaddingVal;
int y2 = rect.BottomRight.Y + boxPaddingVal;
boxes.Add(new BoundingBox(i, x1, y1, x2, y2));
}
}
boxes.Sort((BoundingBox a, BoundingBox b) => { return(a.x - b.x); });
return(boxes);
}
public string Run(Mat image, bool AvoidDNNCrashes)
{
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(image, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
net.SetInput(inputBlob, "data");
if (AvoidDNNCrashes == false)
{
using (var prob = net.Forward("prob"))
{
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
return(String.Format("Best class: #{0} '{1}' Probability: {0:P2}", classId, classNames[classId], classProb));
}
}
}
return("Failed to identify object.");
}
public void LoadMnistTrainingDataFromFile_NetRecognizesAnImageOfA9Correctly()
{
var img_of_9 = Image(Path.Combine("Dnn", "MNIST_9.png"), ImreadModes.Grayscale);
var img9DataBlob = CvDnn.BlobFromImage(img_of_9, 1f / 255.0f);
var modelPath = Path.Combine("_data", "model", "MNISTTest_tensorflow.pb");
var res = -1;
using (var tfGraph = CvDnn.ReadNetFromTensorflow(modelPath))
{
tfGraph.SetInput(img9DataBlob);
using (var prob = tfGraph.Forward())
res = GetResultClass(prob);
}
Assert.True(res == 9);
}
public void Run()
{
const string modelFace = "face-detection-adas-0001.bin";
const string modelFaceTxt = "face-detection-adas-0001.xml";
const string sampleImage = "sample.jpg";
const string outputLoc = "sample_output.jpg";
using var frame = Cv2.ImRead(sampleImage);
int frameHeight = frame.Rows;
int frameWidth = frame.Cols;
using var netFace = CvDnn.ReadNet(modelFace, modelFaceTxt);
netFace.SetPreferableBackend(Backend.INFERENCE_ENGINE);
netFace.SetPreferableTarget(Target.CPU);
using var blob = CvDnn.BlobFromImage(frame, 1.0, new OpenCvSharp.Size(672, 384), new OpenCvSharp.Scalar(0, 0, 0), false, false);
netFace.SetInput(blob);
using (var detection = netFace.Forward())
{
Mat detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F, detection.Ptr(0));
for (int i = 0; i < detectionMat.Rows; i++)
{
float confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
int x1 = (int)(detectionMat.At <float>(i, 3) * frameWidth); //xmin
int y1 = (int)(detectionMat.At <float>(i, 4) * frameHeight); //ymin
int x2 = (int)(detectionMat.At <float>(i, 5) * frameWidth); //xmax
int y2 = (int)(detectionMat.At <float>(i, 6) * frameHeight); //ymax
OpenCvSharp.Rect roi = new OpenCvSharp.Rect(x1, y1, (x2 - x1), (y2 - y1));
roi = AdjustBoundingBox(roi);
Cv2.Rectangle(frame, roi, new Scalar(0, 255, 0), 2, LineTypes.Link4);
}
}
}
var finalOutput = outputLoc;
Cv2.ImWrite(finalOutput, frame);
}
public void Run()
{
const string protoTxt = @"Data/Text/bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"Data/Text/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
var capture = new VideoCapture(0);
Console.Write("Downloading Caffe Model...");
//PrepareModel(caffeModel);
Console.WriteLine("Done");
using (var img = new Mat())
using (var window = new Window("capture"))
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
while (true)
{
capture.Read(img);
if (img.Empty())
{
break;
}
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability: {0:P2}", classProb);
img.PutText(classNames[classId] + String.Format(" Probability: {0:P2}", classProb),
new Point(0 + 10, img.Size().Height - 10), HersheyFonts.HersheyDuplex, 1, Scalar.Black);
window.ShowImage(img);
Cv2.WaitKey(1);
}
}
}
}
private void PerfomrDetection(Net net, string imageFileName)
{
sourceImage = new Mat(imageFileName);
const int MAX_WIDTH = 1200;
const int MAX_HEIGHT = 800;
if (sourceImage.Width > MAX_WIDTH || sourceImage.Height > MAX_HEIGHT)
{
double fx = (double)MAX_WIDTH / sourceImage.Width;
double fy = (double)MAX_HEIGHT / sourceImage.Height;
double ff = fx < fy ? fx : fy;
sourceImage = sourceImage.Resize(new Size(0, 0), ff, ff);
}
App.Current.Dispatcher.Invoke(() =>
{
ImageSource = Imaging.CreateBitmapSourceFromHBitmap(
sourceImage.ToBitmap().GetHbitmap(),
IntPtr.Zero, System.Windows.Int32Rect.Empty,
BitmapSizeOptions.FromEmptyOptions());
});
//setting blob, size can be:320/416/608
//opencv blob setting can check here https://github.com/opencv/opencv/tree/master/samples/dnn#object-detection
var blob = CvDnn.BlobFromImage(sourceImage, 1.0 / 255, new Size(416, 416), new Scalar(), true, false);
net.SetInput(blob);
var outputNames = net.GetUnconnectedOutLayersNames();
//create mats for output layer
detectionOutputs = outputNames.Select(_ => new Mat()).ToArray();
net.Forward(detectionOutputs, outputNames);
DrawResults(detectionOutputs, sourceImage, confidenceThreshold, nmsThreshold);
App.Current.Dispatcher.Invoke(() =>
{
ImageSource = Imaging.CreateBitmapSourceFromHBitmap(
sourceImage.ToBitmap().GetHbitmap(),
IntPtr.Zero, System.Windows.Int32Rect.Empty,
BitmapSizeOptions.FromEmptyOptions());
});
}
private Mat CNN(Mat src)
{
const string configFile = "model/deploy.prototxt";
const string faceModel = "model/res10_300x300_ssd_iter_140000_fp16.caffemodel";
// Read sample image
int srcHeight = src.Rows;
int srcWidth = src.Cols;
using (var faceNet = CvDnn.ReadNetFromCaffe(configFile, faceModel))
{
using (var blob = CvDnn.BlobFromImage(src, 1.0, new OpenCvSharp.Size(300, 300),
new Scalar(104, 117, 123), false, false))
{
faceNet.SetInput(blob, "data");
using (var detection = faceNet.Forward("detection_out"))
{
using (var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F,
detection.Ptr(0)))
{
for (int i = 0; i < detectionMat.Rows; i++)
{
float confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
int x1 = (int)(detectionMat.At <float>(i, 3) * srcWidth);
int y1 = (int)(detectionMat.At <float>(i, 4) * srcHeight);
int x2 = (int)(detectionMat.At <float>(i, 5) * srcWidth);
int y2 = (int)(detectionMat.At <float>(i, 6) * srcHeight);
Cv2.Rectangle(src, new OpenCvSharp.Point(x1, y1), new OpenCvSharp.Point(x2, y2), new Scalar(0, 255, 0), 2,
LineTypes.Link4);
}
}
return(src);
}
}
}
}
}
public IEnumerable <(float Confidence, double CenterX, double CenterY, double Width, double Height)> Detect(Mat image)
{
var blob = CvDnn.BlobFromImage(image, scalarFactor, configSize, blobFromImageMeanParams, true, false);
net.SetInput(blob);
net.Forward(outputLayers, outputNames);
var items = new List <(float Confidence, double CenterX, double CenterY, double Width, double Height)>();
foreach (var prob in outputLayers)
{
for (var i = 0; i < prob.Rows; i++)
{
var confidence = prob.At <float>(i, 4);
if (confidence > Defaults.ConfidenceThreshold)
{
Cv2.MinMaxLoc(prob.Row(i).ColRange(5, prob.Cols), out _, out Point max);
var type = max.X;
var label = labels[type];
var probability = prob.At <float>(i, type + 5);
if (probability > Defaults.ConfidenceThreshold && label.Equals("person"))
{
var centerX = prob.At <float>(i, 0) * image.Width;
var centerY = prob.At <float>(i, 1) * image.Height;
var width = prob.At <float>(i, 2) * image.Width;
var height = prob.At <float>(i, 3) * image.Height;
items.Add((confidence, centerX, centerY, width, height));
}
}
}
}
//return items;
CvDnn.NMSBoxes(
items.Select(i => new Rect((int)(i.CenterX - (i.Width / 2)), (int)(i.CenterY - (i.Height / 2)), (int)i.Width, (int)i.Height)),
items.Select(i => i.Confidence),
(float)Defaults.ConfidenceThreshold,
(float)Defaults.NonMaximaSupressionThreshold,
out int[] indices);
for (int i = 0; i < indices.Length; i++)
{
yield return(items[indices[i]]);
}
}
IEnumerator RecordFrame()
{
yield return(new WaitForEndOfFrame());
var texture = ScreenCapture.CaptureScreenshotAsTexture();
// do something with texture
var matRes = TextureToMat(texture);
// cleanup
Object.Destroy(texture);
// Assign the Vec3b array to Mat
_image.SetArray(0, 0, matRes);
var blob = CvDnn.BlobFromImage(_image, 1 / 255.0, new Size(width, height), new Scalar(), true, false);
net.SetInput(blob, "data");
var prob = net.Forward();
predict(prob);
}
private void btnExcute_Click(object sender, EventArgs e)
{
ctlOpenDialog.Filter = "Pictures |*.jpg";
ctlOpenDialog.InitialDirectory = Environment.CurrentDirectory;
if (ctlOpenDialog.ShowDialog() == DialogResult.OK)
{
ctlPicture.Image = new Bitmap(ctlOpenDialog.FileName);
Mat m = new Mat(ctlOpenDialog.FileName);
Mat blob = CvDnn.BlobFromImage(m, 1, new OpenCvSharp.Size(96, 96), new Scalar(0, 0, 0), false, false);
NeuralNetwork.SetInput(blob);
Mat result = NeuralNetwork.Forward();
OpenCvSharp.Point p1, p2;
Cv2.MinMaxLoc(result, out p1, out p2);
MessageBox.Show("На рисунке представен класс: " + ((Labels)p2.X).ToString());
}
}