public void LoadCaffeModel(String source)
{
Mat img = Cv2.ImRead(source);
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
{
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
// Assert.Equal(1, net.GetLayerId(net.GetLayerNames()[0]));
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1.0, new Size(300, 300), new Scalar(104, 117, 123), false, false))
{
net.SetInput(inputBlob, "data");
using (var detection = net.Forward("detection_out"))
{
// find the best class
Console.WriteLine(detection);
Console.WriteLine(detection.Size(2));
GetMaxClass(detection, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} ", classId);
Console.WriteLine("Probability: {0:P2}", classProb);
// Pause();
//Assert.Equal(812, classId);
}
}
}
}
public override void RunTest()
{
// Read sample image
using var frame = Cv2.ImRead(image);
int frameHeight = frame.Rows;
int frameWidth = frame.Cols;
using var faceNet = CvDnn.ReadNetFromCaffe(configFile, faceModel);
using var blob = CvDnn.BlobFromImage(frame, 1.0, new Size(300, 300), new Scalar(104, 117, 123), false, false);
faceNet.SetInput(blob, "data");
using var detection = faceNet.Forward("detection_out");
using var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F,
detection.Ptr(0));
for (int i = 0; i < detectionMat.Rows; i++)
{
float confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
int x1 = (int)(detectionMat.At <float>(i, 3) * frameWidth);
int y1 = (int)(detectionMat.At <float>(i, 4) * frameHeight);
int x2 = (int)(detectionMat.At <float>(i, 5) * frameWidth);
int y2 = (int)(detectionMat.At <float>(i, 6) * frameHeight);
Cv2.Rectangle(frame, new Point(x1, y1), new Point(x2, y2), new Scalar(0, 255, 0), 2, LineTypes.Link4);
}
}
Window.ShowImages(frame);
}
public override void RunTest()
{
const string protoTxt = @"Data\Text\bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"Data\Text\synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
Console.Write("Downloading Caffe Model...");
PrepareModel(caffeModel);
Console.WriteLine(" Done");
using var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
using var img = new Mat(@"Data\Image\space_shuttle.jpg");
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
// Convert Mat to batch of images
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
net.SetInput(inputBlob, "data");
using var prob = net.Forward("prob");
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability: {0:P2}", classProb);
Console.WriteLine("Press any key to exit");
Console.Read();
}
//private Net Regogniser { get; set; }
//private Net LabelEncoder { get; set; }
/// <summary>
/// Constructor.
/// </summary>
public DnnCaffeFaceDetector(Double MinConfidence, String detectorPath, string protoFile, string detectorModelFile, string embeddingModelFile,
string recogniserModelFile, string labelEncoderFile) : base(MinConfidence)
{
this.DetectorPath = detectorPath;
this.ProtoFile = protoFile;
this.DetectorModelFile = detectorModelFile;
this.EmbeddingModelFile = embeddingModelFile;
this.RecogniserModelFile = recogniserModelFile;
this.LabelEncoderFile = labelEncoderFile;
try
{
// read in the cafe DNN from disk
this.ProtoPath = System.IO.Path.Join(this.DetectorPath, this.ProtoFile);
this.ModelPath = System.IO.Path.Join(this.DetectorPath, this.DetectorModelFile);
this.Detector = CvDnn.ReadNetFromCaffe(this.ProtoPath, this.ModelPath);
// load the serialised face embedding model from disk.
//this.EmbedderPath = System.IO.Path.Join(this.DetectorPath, this.EmbeddingModelFile);
//this.Embedder = CvDnn.ReadNetFromTorch(this.EmbedderPath);
// todo: load facial recognition model and label encoder.
//this.Regogniser = pickle.loads(open(self._recogniser_model_file, "rb").read())
//this.LabelEncoder = pickle.loads(open(self._label_encoder_file, "rb").read())
}
catch (Exception detail)
{
Globals.Log.Error(detail);
}
}
protected override void InitializeModel(string pathModel, string pathConfig)
{
//Initialize caffe model
network = CvDnn.ReadNetFromCaffe(pathConfig, pathModel);
this.Scale = 1;
}
public void LoadCaffeModel()
{
const string protoTxt = @"_data/text/bvlc_googlenet.prototxt";
const string caffeModelUrl = "https://drive.google.com/uc?id=1RUsoiLiXrKBQu9ibwsMqR3n_UkhnZLRR"; //"http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel";
const string caffeModel = "_data/model/bvlc_googlenet.caffemodel";
const string synsetWords = @"_data/text/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
testOutputHelper.WriteLine("Downloading Caffe Model...");
PrepareModel(caffeModelUrl, caffeModel);
testOutputHelper.WriteLine("Done");
using var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
//Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
var layerName = net.GetLayerNames()[0];
Assert.NotNull(layerName);
Assert.Equal(1, net.GetLayerId(layerName !));
// Convert Mat to batch of images
using var img = Image(@"space_shuttle.jpg");
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
net.SetInput(inputBlob, "data");
using var prob = net.Forward("prob");
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
testOutputHelper.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
testOutputHelper.WriteLine("Probability: {0:P2}", classProb);
Pause();
Assert.Equal(812, classId);
}
private string OpenCVDeepLearningDetector(string path)
{
// uses emugu library
//https://medium.com/@vinuvish/face-detection-with-opencv-and-deep-learning-90bff9028fa8
string prototextPath = @"./Dnn/deploy.prototxt";
string caffeModelPath = @"./Dnn/res10_300x300_ssd_iter_140000.caffemodel";
// load the model;
var net = CvDnn.ReadNetFromCaffe(prototxt: prototextPath, caffeModel: caffeModelPath);
// get the image
OpenCvSharp.Mat image = Cv2.ImRead(path);
// get the original image size
OpenCvSharp.Size imageSize = image.Size();
// the dnn detector works on a 300x300 image;
// now resize the image for the Dnn dector;
OpenCvSharp.Size size = new OpenCvSharp.Size(299, 299);
image = image.Resize(size);
// set the scalar property to RGB colors, don't know what these values represent.
OpenCvSharp.Scalar mcvScalar = new OpenCvSharp.Scalar(104.0, 177.0, 123.0);
var blob = CvDnn.BlobFromImage(image: image, scaleFactor: 1, size: size, mean: mcvScalar, swapRB: true);
net.SetInput(blob);
OpenCvSharp.Mat detections = net.Forward();
// convert the detected values to a faces object that we can use to
// draw rectangles.
List <ConfidenceRect> Faces = new List <ConfidenceRect>();
//var rows = detections.SizeOfDimension[2];
//Array ans = detections.GetData();
//for (int n = 0; n < rows; n++)
//{
// object confidence = ans.GetValue(0, 0, n, 2);
// object x1 = ans.GetValue(0, 0, n, 3);
// object y1 = ans.GetValue(0, 0, n, 4);
// object x2 = ans.GetValue(0, 0, n, 5);
// object y2 = ans.GetValue(0, 0, n, 6);
// ConfidenceRect cr = new ConfidenceRect(confidence, x1, y1, x2, y2, imageSize);
// if (cr.Confidence > 0)
// {
// Debug.WriteLine($"Confidence {cr.Confidence}");
// }
// if (cr.Confidence > Confidence)
// {
// Faces.Add(cr);
// }
//}
//// convert to a writeableBitmap
//WriteableBitmap writeableBitmap = new WriteableBitmap(ImageSource);
//ImageSource = ConvertWriteableBitmapToBitmapImage(writeableBitmap);
//OnPropertyChanged("ImageSource");
//DrawDnnOnImage?.Invoke(Faces, imageSize);
//return Faces.Count.ToString();
return(null);
}
public CaffeDnnFaceDetector(string modelFile)
{
var modelDirectory = Path.GetDirectoryName(modelFile) !;
var configFile = Path.Combine(modelDirectory, "deploy.prototxt");
Net = CvDnn.ReadNetFromCaffe(configFile, modelFile);
Net.SetPreferableBackend(Net.Backend.INFERENCE_ENGINE);
Net.SetPreferableTarget(Net.Target.OPENCL_FP16);
}
static void Main(string[] args)
{
const string prototext = @"..\..\..\..\data\bvlc_googlenet.prototxt";
const string caffeModel = @"..\..\..\..\data\bvlc_googlenet.caffemodel";
const string synsetWords = @"..\..\..\..\data\synset_words.txt";
string[] classNames = File.ReadAllLines(synsetWords).Select(l => l.Split(' ').Last()).ToArray();
//Use stopwatch object fro timing of the operation
Stopwatch sw = new Stopwatch();
string imgPath = @"D:\DeepLearningOpenCV\images\DogBycleCar.jpg";
using (var net = CvDnn.ReadNetFromCaffe(prototext, caffeModel))
using (var img = Cv2.ImRead(imgPath))
{
//Just out of curiosity, I wanted to get the Layer names of the NN Construct
// by calling GetLayerNames method of the Net object
string[] layerNames = net.GetLayerNames();
Console.WriteLine("Layer names : {0}", string.Join(", ", layerNames));
Console.WriteLine();
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123), swapRB: true, crop: false))
{
sw.Start();
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
sw.Stop();
Console.WriteLine($"Cost of calculating prob {sw.ElapsedMilliseconds} ms");
int cols = prob.Cols;
int rows = prob.Rows;
Console.WriteLine("Cols: " + cols + ", Rows:" + rows);
// GetMaxProClass(prob, out int classId, out double classProb);
Cv2.MinMaxLoc(prob, out _, out double classProb, out _, out Point classNumberPoint);
int classId = classNumberPoint.X;
Console.WriteLine("Best class: #{0}, '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability:{0:P2}", classProb);
string txt = "Label: " + classNames[classId] + ", % " + (100 * classProb).ToString("0.####");
Cv2.PutText(img, txt, new Point(5, 25), HersheyFonts.HersheySimplex, 0.7, new Scalar(0, 0, 255), 2);
//Cv2.ImWrite("classification.jpg", img);
Cv2.ImShow("image", img);
}
}
}
Cv2.WaitKey();
Cv2.DestroyAllWindows();
// Console.Write("Downloading Caffe Model...");
Console.WriteLine("Press any key to exit");
Console.Read();
}
public void initialize(string protoTxt, string caffeModel, string synsetWords)
{
classNames = File.ReadAllLines(synsetWords).Select(line => line.Split(' ').Last()).ToArray();
PrepareModel(caffeModel);
net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
Console.WriteLine("Preparation complete");
}
public CaffeDnnFaceDetector(string modelFile)
{
var configFile = Path.ChangeExtension(modelFile, ".prototxt") !;
Net = CvDnn.ReadNetFromCaffe(configFile, modelFile);
//Cv2.SetNumThreads(2);
//Net.SetPreferableBackend(Net.Backend.OPENCV);
Net.SetPreferableBackend(Net.Backend.INFERENCE_ENGINE);
Net.SetPreferableTarget(Net.Target.MYRIAD);
//Net.SetPreferableTarget(Net.Target.OPENCL);
}
static void Main()
{
var file = "bali.jpg";
var prototxt = "deploy.prototxt";
var model = "VGG_VOC0712Plus_SSD_512x512_ft_iter_160000.caffemodel";
var colors = Enumerable.Repeat(false, 21).Select(x => Scalar.RandomColor()).ToArray();
//get image
var org = Cv2.ImRead(file);
var blob = CvDnn.BlobFromImage(org, 1, new Size(512, 512));
//setup model
var net = CvDnn.ReadNetFromCaffe(prototxt, model);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward("detection_out");
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
//reshape from [1,1,200,7] to [200,7]
var p = prob.Reshape(1, prob.Size(2));
for (int i = 0; i < prob.Size(2); i++)
{
var confidence = p.At <float>(i, 2);
if (confidence > 0.4)
{
//get value what we need
var idx = (int)p.At <float>(i, 1);
var w1 = (int)(org.Width * p.At <float>(i, 3));
var h1 = (int)(org.Width * p.At <float>(i, 4));
var w2 = (int)(org.Width * p.At <float>(i, 5));
var h2 = (int)(org.Width * p.At <float>(i, 6));
var label = $"{Labels[idx]} {confidence * 100:0.00}%";
Console.WriteLine($"{label}");
//draw result
Cv2.Rectangle(org, new Rect(w1, h1, w2 - w1, h2 - h1), colors[idx], 2);
var textSize = Cv2.GetTextSize(label, HersheyFonts.HersheyTriplex, 0.5, 1, out var baseline);
Cv2.Rectangle(org, new Rect(new Point(w1, h1 - textSize.Height),
new Size(textSize.Width, textSize.Height + baseline)), colors[idx], Cv2.FILLED);
Cv2.PutText(org, label, new Point(w1, h1), HersheyFonts.HersheyTriplex, 0.5, Scalar.Black);
}
}
using (new Window("image", org))
{
Cv2.WaitKey();
}
}
static void Main()
{
//model can get from http://www.robots.ox.ac.uk/~vgg/software/vgg_face/
var file = "jeremy-clarkson-v2.jpg";
//var file = "john-cena.jpg";
var prototxt = "VGG_FACE_deploy.prototxt";
var model = "VGG_FACE.caffemodel";
var labeltxt = "names.txt";
//read all names
var labels = ReadLabels(labeltxt);
var org = Cv2.ImRead(file);
var blob = CvDnn.BlobFromImage(org, 1, new Size(224, 224));
var net = CvDnn.ReadNetFromCaffe(prototxt, model);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward("prob");
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
//convert result to list
var probList = new Dictionary <int, float>();
for (int i = 0; i < prob.Width; i++)
{
probList.Add(i, prob.At <float>(0, i));
}
//get top 3
var top3 = probList.OrderByDescending(x => x.Value).Take(3).ToList();
foreach (var result in top3)
{
Console.WriteLine($"{labels[result.Key]}:{result.Value*100:0.00}%");
}
//draw result
org.PutText($"{labels[top3.First().Key]}:{top3.First().Value * 100:0.00}%",
new Point(0, 25), HersheyFonts.HersheyTriplex, 1, Scalar.OrangeRed);
using (new Window("image", org))
{
Cv2.WaitKey();
}
}
private CaffeData LoadCaffeModel()
{
const string protoTxt = @"_data/text/bvlc_googlenet.prototxt";
const string caffeModelUrl = "https://drive.google.com/uc?id=1RUsoiLiXrKBQu9ibwsMqR3n_UkhnZLRR"; //"http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel";
const string caffeModel = "_data/model/bvlc_googlenet.caffemodel";
const string synsetWords = @"_data/text/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
ModelDownloader.DownloadAndSave(new Uri(caffeModelUrl), caffeModel);
var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
Assert.NotNull(net);
return(new CaffeData(net !, classNames));
}
public void Run()
{
const string protoTxt = @"Data/Text/bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"Data/Text/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
var capture = new VideoCapture(0);
Console.Write("Downloading Caffe Model...");
//PrepareModel(caffeModel);
Console.WriteLine("Done");
using (var img = new Mat())
using (var window = new Window("capture"))
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
while (true)
{
capture.Read(img);
if (img.Empty())
{
break;
}
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability: {0:P2}", classProb);
img.PutText(classNames[classId] + String.Format(" Probability: {0:P2}", classProb),
new Point(0 + 10, img.Size().Height - 10), HersheyFonts.HersheyDuplex, 1, Scalar.Black);
window.ShowImage(img);
Cv2.WaitKey(1);
}
}
}
}
private Mat CNN(Mat src)
{
const string configFile = "model/deploy.prototxt";
const string faceModel = "model/res10_300x300_ssd_iter_140000_fp16.caffemodel";
// Read sample image
int srcHeight = src.Rows;
int srcWidth = src.Cols;
using (var faceNet = CvDnn.ReadNetFromCaffe(configFile, faceModel))
{
using (var blob = CvDnn.BlobFromImage(src, 1.0, new OpenCvSharp.Size(300, 300),
new Scalar(104, 117, 123), false, false))
{
faceNet.SetInput(blob, "data");
using (var detection = faceNet.Forward("detection_out"))
{
using (var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F,
detection.Ptr(0)))
{
for (int i = 0; i < detectionMat.Rows; i++)
{
float confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
int x1 = (int)(detectionMat.At <float>(i, 3) * srcWidth);
int y1 = (int)(detectionMat.At <float>(i, 4) * srcHeight);
int x2 = (int)(detectionMat.At <float>(i, 5) * srcWidth);
int y2 = (int)(detectionMat.At <float>(i, 6) * srcHeight);
Cv2.Rectangle(src, new OpenCvSharp.Point(x1, y1), new OpenCvSharp.Point(x2, y2), new Scalar(0, 255, 0), 2,
LineTypes.Link4);
}
}
return(src);
}
}
}
}
}
public void Run()
{
const string configFile = "deploy.prototxt";
const string faceModel = "res10_300x300_ssd_iter_140000_fp16.caffemodel";
const string finalOutput = "DetectedFaces.jpg";
const string image = "faces.jpg";
// Read sample image
using var frame = Cv2.ImRead(image);
int frameHeight = frame.Rows;
int frameWidth = frame.Cols;
using var faceNet = CvDnn.ReadNetFromCaffe(configFile, faceModel);
using var blob = CvDnn.BlobFromImage(frame, 1.0, new Size(300, 300),
new Scalar(104, 117, 123), false, false);
faceNet.SetInput(blob, "data");
using var detection = faceNet.Forward("detection_out");
using var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F,
detection.Ptr(0));
for (int i = 0; i < detectionMat.Rows; i++)
{
float confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
int x1 = (int)(detectionMat.At <float>(i, 3) * frameWidth);
int y1 = (int)(detectionMat.At <float>(i, 4) * frameHeight);
int x2 = (int)(detectionMat.At <float>(i, 5) * frameWidth);
int y2 = (int)(detectionMat.At <float>(i, 6) * frameHeight);
Cv2.Rectangle(frame, new Point(x1, y1), new Point(x2, y2), new Scalar(0, 255, 0), 2,
LineTypes.Link4);
}
}
Cv2.ImWrite(finalOutput, frame);
}
public async Task LoadCaffeModel()
{
const string protoTxt = @"_data\text\bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"_data\text\synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
Console.Write("Downloading Caffe Model...");
await PrepareModel(caffeModel);
Console.WriteLine(" Done");
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
using (var img = Image(@"space_shuttle.jpg"))
{
//Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Assert.Equal(1, net.GetLayerId(net.GetLayerNames()[0]));
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability: {0:P2}", classProb);
Pause();
Assert.Equal(812, classId);
}
}
}
}
public void Run()
{
const string protoTxt = @"Data/bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"Data/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
Console.Write("Downloading Caffe Model...");
PrepareModel(caffeModel);
Console.WriteLine(" Done");
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
using (var img = new Mat(@"Data/Birds.jpg"))
{
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
// find the best class
//GetMaxClass(prob, out int classId, out double classProb);
// int classId;
double classProb;
//Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
//Console.WriteLine("Probability: {0:P2}", classProb);
using (Mat probMat = prob.Reshape(1, 1))
{
Cv2.MinMaxLoc(probMat, out _, out classProb, out _, out var classNumber);
int classId = classNumber.X;
Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability: {0:P2}", classProb);
Console.WriteLine("Columns Please: {0}", probMat.Cols);
Console.WriteLine("1. For the snowbird: {0}", probMat.Get <double>(13, 0));
Console.WriteLine("2. For the snowbird: {0:P2}", probMat.Get <double>(0, 13));
for (int i = 0; i < probMat.GetArray(0, 13).Length; i++)
{
Console.WriteLine("Here we go: {0:P2}", probMat.GetArray(0, 13)[i]);
}
Console.WriteLine("3. For the snowbird: {0}", probMat.GetArray(0, 13)[0]);
for (int i = 0; i < probMat.Cols; i++)
{
classId = i;
classProb = probMat.At <double>(0, i);
double probInt = probMat.At <double>(i, 0);
Console.WriteLine("Close classes: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Prob Int: {0}", probInt);
Console.WriteLine("Probability: {0} \n", classProb);
}
}
}
}
}
}
private Rect[] GetDnnFaces(Mat frame, int maxFaces)
{
//double ScaleFactor = 1.28;
double Confidence = 0.9;
//int Neighbors = 2;
System.Drawing.Size imageSize = new System.Drawing.Size(frame.Width, frame.Height);
string prototextPath = @"./Dnn/deploy.prototxt";
string caffeModelPath = @"./Dnn/res10_300x300_ssd_iter_140000.caffemodel";
if (!File.Exists(prototextPath) || !File.Exists(caffeModelPath))
{
return(null);
}
// load the model;
//var net = DnnInvoke.ReadNetFromCaffe(prototxt: prototextPath, caffeModel: caffeModelPath);
var net = CvDnn.ReadNetFromCaffe(prototxt: prototextPath, caffeModel: caffeModelPath);
// get the image
// the dnn detector works on a 300x300 image;
int targetWidth = 300;
int targetHeight = 300;
// now resize the image for the Dnn dector;
Mat image = frame.Resize(new OpenCvSharp.Size(targetWidth, targetHeight), 0, 0, InterpolationFlags.Lanczos4);
// (targetWidth, targetHeight, Emgu.CV.CvEnum.Inter.Lanczos4);
OpenCvSharp.Size size = new OpenCvSharp.Size(targetWidth, targetHeight);
var blob = CvDnn.BlobFromImage(image: image, scaleFactor: 1, size: size);
//CvDnn.BlobFromImage(image: image, scaleFactor: ScaleFactor, size: size, mean: mcvScalar, swapRB: true);
net.SetInput(blob);
Mat detections = net.Forward();
List <ConfidenceRect> faces = new List <ConfidenceRect>();
for (int i = 0; i < detections.Rows; i++)
{
float confidence = detections.At <float>(1, 2);
if (confidence < Confidence)
{
continue;
}
int x_left_bottom = (int)(detections.At <float>(i, 3));
int y_left_bottom = (int)(detections.At <float>(i, 4));
int x_right_top = (int)(detections.At <float>(i, 5));
int y_right_top = (int)(detections.At <float>(i, 6));
faces.Add(new ConfidenceRect(confidence, x_left_bottom,
y_left_bottom,
x_right_top - x_left_bottom,
y_right_top - y_left_bottom,
imageSize));
}
if (faces.Count > 0)
{
System.Drawing.Rectangle ans = faces.OrderBy(n => n.Confidence).First().AsRectangle();
return(new OpenCvSharp.Rect[] { new OpenCvSharp.Rect(ans.X, ans.Y, ans.Width, ans.Height) });
}
image.Dispose();
detections.Dispose();
return(null);
}
public void Detect()
{
Centroid ct = new Centroid();
var objects = new Dictionary <int, (double x, double y)>();
var prototxt = @".\model\deploy.prototxt";
var model = @".\model\res10_300x300_ssd_iter_140000.caffemodel";
var colors = Enumerable.Repeat(false, 21).Select(x => Scalar.RandomColor()).ToArray();
double fps = 10;
int sleepTime = (int)Math.Round(1000 / fps);
VideoCapture capture = new VideoCapture(0);
//capture.Open(0);
//Console.WriteLine("Model is Loading");
var net = CvDnn.ReadNetFromCaffe(prototxt, model);
//using (Window window = new Window("capture"))
using (Mat frame = new Mat())
{
while (true)
{
capture.Read(frame);
if (frame.Empty())
{
break;
}
Cv2.Resize(frame, frame, new Size(400, 300));
double H = frame.Height;
double W = frame.Width;
var meanC = new Scalar(104.0, 177.0, 123.0);
var blob = CvDnn.BlobFromImage(frame, 1, new OpenCvSharp.Size(W, H), meanC, true, false);
net.SetInput(blob, "data");
var prob = net.Forward("detection_out");
List <List <double> > rects = new List <List <double> >();
List <double> box = new List <double>();
//var p = prob.Reshape(1, prob.Size(2));
for (int i = 0; i < prob.Size(2); i++)
{
var confidence = prob.At <float>(0, 0, i, 2);
if (confidence > 0.9)
{
//get value what we need
var idx = prob.At <float>(0, 0, i, 1);
var w1 = (W * prob.At <float>(0, 0, i, 3));
var h1 = (H * prob.At <float>(0, 0, i, 4));
var w2 = (W * prob.At <float>(0, 0, i, 5));
var h2 = (H * prob.At <float>(0, 0, i, 6));
rects.Add(new List <double> {
w1, h1, w2, h2
});
//System.Diagnostics.Debug.WriteLine(rectCount++.ToString());
Cv2.Rectangle(frame, new Rect((int)w1, (int)h1, (int)w2 - (int)w1, (int)h2 - (int)h1), Scalar.Green, 2);
}
}
objects = ct.Update(rects);
//System.Diagnostics.Debug.WriteLine($"Rect Count {rects.Count}");
foreach (var item in objects)
{
string text = "ID:" + item.Key.ToString();
var textSize = Cv2.GetTextSize(text, HersheyFonts.HersheyTriplex, 0.2, 1, out var baseline);
Cv2.PutText(frame, text, new OpenCvSharp.Point(item.Value.x - 10, item.Value.y - 10), HersheyFonts.HersheyTriplex, 0.75, Scalar.Green, 2);
Cv2.Circle(frame, (int)item.Value.x, (int)item.Value.y, 2, Scalar.Green, 1);
}
using (new Window("capture", frame))
//window.ShowImage(frame);
Cv2.WaitKey(sleepTime);
//frame.Release();
//frame.Dispose();
}
}
}
public async Task <IEnumerable <Face> > ProcessAsync(string inputFilename)
{
if (!File.Exists(inputFilename))
{
throw new FileNotFoundException(nameof(inputFilename));
}
// Read sample image
using var frame = Cv2.ImRead(inputFilename);
var frameHeight = frame.Rows;
var frameWidth = frame.Cols;
using var faceNet = CvDnn.ReadNetFromCaffe(ConfigFile, FaceModel);
using var blob = CvDnn.BlobFromImage(
frame,
1.0,
new Size(300, 300),
new Scalar(104, 117, 123),
false,
false);
faceNet.SetInput(blob, "data");
using var detection = faceNet.Forward("detection_out");
using var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F, detection.Ptr(0));
var list = new List <ConfidenceBox>(detectionMat.Rows);
for (var i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
var x1 = (int)(detectionMat.At <float>(i, 3) * frameWidth);
var y1 = (int)(detectionMat.At <float>(i, 4) * frameHeight);
var x2 = (int)(detectionMat.At <float>(i, 5) * frameWidth);
var y2 = (int)(detectionMat.At <float>(i, 6) * frameHeight);
list.Add(new ConfidenceBox(new Point(x1, y1), new Point(x2, y2), confidence));
}
// var orderedFaces = list.OrderByDescending(x => x.Confidence).Where(x => x.Confidence > 0.3).ToList();
// var origFilename = new FileInfo(inputFilename).Name;
// var faces = orderedFaces;
// foreach (var face in faces)
// {
// FaceBoxer.Draw(frame, face.P1, face.P2, face.Confidence);
// }
// var outputFilename = Path.Combine(outputDirectory, $"{origFilename}_{Name}.jpg");
// Cv2.ImWrite(outputFilename, frame);
// for (var i = 0; i < orderedList.Count; i++)
// {
// var box = orderedList[i];
// FaceBoxer.Draw(frame, box.P1, box.P2, box.Confidence);
// var outputFilename = Path.Combine(outputDirectory, $"{origFilename}_{Name}_{i+1}.jpg");
// Cv2.ImWrite(outputFilename, frame);
// }
await Task.Yield();
return(list.Select(x => new Face
{
Position = new RectangleDto
{
Top = x.P1.Y,
Right = x.P2.X,
Left = x.P1.X,
Bottom = x.P2.Y,
}.ToRectangle(),
Confidence = x.Confidence,
})
.ToList());
}
async void PushImageStream()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
Console.WriteLine("run bash shell: 'sudo modprobe bcm2835-v4l2'");
"sudo modprobe bcm2835-v4l2".Bash();
}
CascadeClassifier cascadeClassifier = new CascadeClassifier("haarcascade_frontalface_default.xml");
// Opens MP4 file (ffmpeg is probably needed)
var capture = new VideoCapture(0);
int sleepTime = (int)Math.Round(1000 / ((capture.Fps == 0) ? 60 : capture.Fps));
var hog = new HOGDescriptor();
hog.SetSVMDetector(HOGDescriptor.GetDefaultPeopleDetector());
var recognizer = OpenCvSharp.Face.EigenFaceRecognizer.Create(2);
var img1 = Cv2.ImRead(SamplesCore.FilePath.Image.Girl, ImreadModes.GrayScale);
var img2 = Cv2.ImRead(SamplesCore.FilePath.Image.Lenna, ImreadModes.GrayScale);
System.Console.WriteLine("img1 width: " + img1.Size().Width + " height:" + img1.Size().Height);
System.Console.WriteLine("img2 width: " + img2.Size().Width + " height:" + img2.Size().Height);
recognizer.Train(new List <Mat>()
{
img1, img2
}, new List <int>()
{
1, 2
});
recognizer.Write("trainfile1.dat");
recognizer.Read("trainfile1.dat");
System.Console.WriteLine("before predict");
System.Console.WriteLine("img1 width: " + img1.Size().Width + " height:" + img1.Size().Height);
System.Console.WriteLine("img2 width: " + img2.Size().Width + " height:" + img2.Size().Height);
recognizer.Predict(InputArray.Create(img1), out int label, out double confidence);
System.Console.WriteLine("label: " + label + " confidence: " + confidence);
const string protoTxt = @"Data/Text/bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"Data/Text/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
//using (var window = new Window("capture"))
{
// Frame image buffer
Mat image = new Mat();
Task.Run(() =>
{
while (true)
{
FaceProcessing(image, cascadeClassifier);
}
});
Task.Run(() =>
{
while (true)
{
HogProcessing(image, hog);
}
});
Task.Run(() =>
{
while (true)
{
CaffeProcessing(image, net, classNames);
}
});
// When the movie playback reaches end, Mat.data becomes NULL.
while (true)
{
capture.Read(image); // same as cvQueryFrame
if (image.Empty())
{
break;
}
currentMat = image.Clone();
foreach (var rect in currentFaces)
{
Cv2.Rectangle(image, rect, Scalar.Red, 2);
}
foreach (var rect in currentBodies)
{
Cv2.Rectangle(image, rect, Scalar.Green, 2);
}
image.PutText(classNames[this.classId] + String.Format(" Probability: {0:P2}", this.classProb),
new Point(0 + 10, image.Size().Height - 10), HersheyFonts.HersheyDuplex, 1, Scalar.White);
var bytes = image.ToMemoryStream().ToArray();
await this.socket.SendAsync(new ArraySegment <byte>(bytes, 0, bytes.Length), WebSocketMessageType.Binary, true, CancellationToken.None);
//if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
//{
// window.ShowImage(image);
// Cv2.WaitKey(sleepTime);
//}
}
}
}
public override void RunTest()
{
const string model = "pose_iter_102000.caffemodel";
const string modelTxt = "pose_deploy.prototxt";
const string sampleImage = "hand.jpg";
const string outputLoc = "Output_Hand.jpg";
const int nPoints = 22;
const double thresh = 0.01;
int[][] posePairs =
{
new[] { 0, 1 }, new[] { 1, 2 }, new[] { 2, 3 }, new[] { 3, 4 }, //thumb
new[] { 0, 5 }, new[] { 5, 6 }, new[] { 6, 7 }, new[] { 7, 8 }, //index
new[] { 0, 9 }, new[] { 9, 10 }, new[] { 10, 11 }, new[] { 11, 12 }, //middle
new[] { 0, 13 }, new[] { 13, 14 }, new[] { 14, 15 }, new[] { 15, 16 }, //ring
new[] { 0, 17 }, new[] { 17, 18 }, new[] { 18, 19 }, new[] { 19, 20 }, //small
};
using var frame = Cv2.ImRead(sampleImage);
using var frameCopy = frame.Clone();
int frameWidth = frame.Cols;
int frameHeight = frame.Rows;
float aspectRatio = frameWidth / (float)frameHeight;
int inHeight = 368;
int inWidth = ((int)(aspectRatio * inHeight) * 8) / 8;
using var net = CvDnn.ReadNetFromCaffe(modelTxt, model);
using var inpBlob = CvDnn.BlobFromImage(frame, 1.0 / 255, new Size(inWidth, inHeight),
new Scalar(0, 0, 0), false, false);
net.SetInput(inpBlob);
using var output = net.Forward();
int H = output.Size(2);
int W = output.Size(3);
var points = new List <Point>();
for (int n = 0; n < nPoints; n++)
{
// Probability map of corresponding body's part.
using var probMap = new Mat(H, W, MatType.CV_32F, output.Ptr(0, n));
Cv2.Resize(probMap, probMap, new Size(frameWidth, frameHeight));
Cv2.MinMaxLoc(probMap, out _, out var maxVal, out _, out var maxLoc);
if (maxVal > thresh)
{
Cv2.Circle(frameCopy, maxLoc.X, maxLoc.Y, 8, new Scalar(0, 255, 255), -1,
LineTypes.Link8);
Cv2.PutText(frameCopy, Cv2.Format(n), new OpenCvSharp.Point(maxLoc.X, maxLoc.Y),
HersheyFonts.HersheyComplex, 1, new Scalar(0, 0, 255), 2, LineTypes.AntiAlias);
}
points.Add(maxLoc);
}
int nPairs = 20; //(POSE_PAIRS).Length / POSE_PAIRS[0].Length;
for (int n = 0; n < nPairs; n++)
{
// lookup 2 connected body/hand parts
Point partA = points[posePairs[n][0]];
Point partB = points[posePairs[n][1]];
if (partA.X <= 0 || partA.Y <= 0 || partB.X <= 0 || partB.Y <= 0)
{
continue;
}
Cv2.Line(frame, partA, partB, new Scalar(0, 255, 255), 8);
Cv2.Circle(frame, partA.X, partA.Y, 8, new Scalar(0, 0, 255), -1);
Cv2.Circle(frame, partB.X, partB.Y, 8, new Scalar(0, 0, 255), -1);
}
var finalOutput = outputLoc;
Cv2.ImWrite(finalOutput, frame);
}
public override void RunTest()
{
const string model = "pose_iter_160000.caffemodel";
const string modelTxt = "pose_deploy_linevec_faster_4_stages.prototxt";
const string sampleImage = "single.jpeg";
const string outputLoc = "Output-Skeleton.jpg";
const int nPoints = 15;
const double thresh = 0.1;
int[][] posePairs =
{
new[] { 0, 1 }, new[] { 1, 2 }, new[] { 2, 3 },
new[] { 3, 4 }, new[] { 1, 5 }, new[] { 5, 6 },
new[] { 6, 7 }, new[] { 1, 14 }, new[] { 14, 8 },new[] { 8, 9 },
new[] { 9, 10 }, new[] { 14, 11 }, new[] { 11, 12 },new[] { 12, 13 },
};
using var frame = Cv2.ImRead(sampleImage);
using var frameCopy = frame.Clone();
int frameWidth = frame.Cols;
int frameHeight = frame.Rows;
const int inWidth = 368;
const int inHeight = 368;
using var net = CvDnn.ReadNetFromCaffe(modelTxt, model);
net.SetPreferableBackend(Backend.OPENCV);
net.SetPreferableTarget(Target.CPU);
using var inpBlob = CvDnn.BlobFromImage(frame, 1.0 / 255, new Size(inWidth, inHeight), new Scalar(0, 0, 0), false, false);
net.SetInput(inpBlob);
using var output = net.Forward();
int H = output.Size(2);
int W = output.Size(3);
var points = new List <Point>();
for (int n = 0; n < nPoints; n++)
{
// Probability map of corresponding body's part.
using var probMap = new Mat(H, W, MatType.CV_32F, output.Ptr(0, n));
var p = new Point2f(-1, -1);
Cv2.MinMaxLoc(probMap, out _, out var maxVal, out _, out var maxLoc);
var x = (frameWidth * maxLoc.X) / W;
var y = (frameHeight * maxLoc.Y) / H;
if (maxVal > thresh)
{
p = maxLoc;
p.X *= (float)frameWidth / W;
p.Y *= (float)frameHeight / H;
Cv2.Circle(frameCopy, (int)p.X, (int)p.Y, 8, new Scalar(0, 255, 255), -1);
Cv2.PutText(frameCopy, Cv2.Format(n), new Point((int)p.X, (int)p.Y), HersheyFonts.HersheyComplex, 1, new Scalar(0, 0, 255), 2);
}
points.Add((Point)p);
}
int nPairs = 14; //(POSE_PAIRS).Length / POSE_PAIRS[0].Length;
for (int n = 0; n < nPairs; n++)
{
// lookup 2 connected body/hand parts
Point partA = points[posePairs[n][0]];
Point partB = points[posePairs[n][1]];
if (partA.X <= 0 || partA.Y <= 0 || partB.X <= 0 || partB.Y <= 0)
{
continue;
}
Cv2.Line(frame, partA, partB, new Scalar(0, 255, 255), 8);
Cv2.Circle(frame, partA.X, partA.Y, 8, new Scalar(0, 0, 255), -1);
Cv2.Circle(frame, partB.X, partB.Y, 8, new Scalar(0, 0, 255), -1);
}
var finalOutput = outputLoc;
Cv2.ImWrite(finalOutput, frame);
}
static void Main()
{
var file = "best-supporting-actress.jpg";
//var file = "oscars-2017.jpg";
var prototxt = "VGG_FACE_deploy.prototxt";
var model = "VGG_FACE.caffemodel";
var labeltxt = "names.txt";
var cascade = "haarcascade_frontalface_default.xml";
var org = Cv2.ImRead(file);
//get face using haarcascades , https://github.com/opencv/opencv/tree/master/data/haarcascades
var faceCascade = new CascadeClassifier();
faceCascade.Load(cascade);
var faces = faceCascade.DetectMultiScale(org, 1.1, 6, HaarDetectionType.DoRoughSearch, new Size(60, 60));
var faceList = new List <Mat>();
foreach (var rect in faces)
{
Cv2.Rectangle(org, rect, Scalar.Red);
faceList.Add(org[rect]);
}
//read all names
var labels = ReadLabels(labeltxt);
var blob = CvDnn.BlobFromImages(faceList, 1, new Size(224, 224));
var net = CvDnn.ReadNetFromCaffe(prototxt, model);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward("prob");
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
for (int n = 0; n < prob.Height; n++)
{
//convert result to list
var probList = new Dictionary <int, float>();
for (int i = 0; i < prob.Width; i++)
{
probList.Add(i, prob.At <float>(n, i));
}
//get top 1
var top1 = probList.OrderByDescending(x => x.Value).First();
var label = $"{labels[top1.Key]}:{top1.Value * 100:0.00}%";
Console.WriteLine(label);
//show if confidence > 50%
if (top1.Value > 0.5)
{
var textsize = Cv2.GetTextSize(label, HersheyFonts.HersheyTriplex, 0.5, 1, out var baseline);
var y = faces[n].TopLeft.Y - textsize.Height - baseline <= 0
? faces[n].BottomRight.Y + textsize.Height + baseline : faces[n].TopLeft.Y - baseline;
//draw result
org.PutText(label, new Point(faces[n].TopLeft.X, y), HersheyFonts.HersheyTriplex, 0.5, Scalar.OrangeRed);
}
}
using (new Window("image", org))
{
Cv2.WaitKey();
}
}
