/// <summary>
/// Begin detect pass the blob to the dnn network
/// </summary>
/// <param name="img">Image to detect</param>
/// <param name="minProbability">Min probability to get label</param>
/// <param name="labelsFilters">Label to filters</param>
/// <returns>Net Result</returns>
protected override NetResult[] BeginDetect(Bitmap img, float minProbability = 0.3F, string[] labelsFilters = null)
{
using (Mat mat = OpenCvSharp.Extensions.BitmapConverter.ToMat(img))
{
//Create a blob
var blob = CvDnn.BlobFromImage(mat, Scale, size: new OpenCvSharp.Size(256, 256), mean: new Scalar(78.4263377603, 87.7689143744, 114.895847746), swapRB: false);
//Set the input for the layer "data"
network.SetInput(blob, "data");
//Get result of the layer prob
var prop = network.Forward("prob");
//Get the maxLoc and max for probability and the label index
prop.MinMaxLoc(out double min, out double max, out OpenCvSharp.Point minLoc, out OpenCvSharp.Point maxLoc);
return(new NetResult[]
{
new NetResult()
{
Label = Labels[maxLoc.X],
Probability = max
}
});
}
}
public override void RunTest()
{
// Read sample image
using var frame = Cv2.ImRead(image);
int frameHeight = frame.Rows;
int frameWidth = frame.Cols;
using var faceNet = CvDnn.ReadNetFromCaffe(configFile, faceModel);
using var blob = CvDnn.BlobFromImage(frame, 1.0, new Size(300, 300), new Scalar(104, 117, 123), false, false);
faceNet.SetInput(blob, "data");
using var detection = faceNet.Forward("detection_out");
using var detectionMat = new Mat(detection.Size(2), detection.Size(3), MatType.CV_32F,
detection.Ptr(0));
for (int i = 0; i < detectionMat.Rows; i++)
{
float confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
int x1 = (int)(detectionMat.At <float>(i, 3) * frameWidth);
int y1 = (int)(detectionMat.At <float>(i, 4) * frameHeight);
int x2 = (int)(detectionMat.At <float>(i, 5) * frameWidth);
int y2 = (int)(detectionMat.At <float>(i, 6) * frameHeight);
Cv2.Rectangle(frame, new Point(x1, y1), new Point(x2, y2), new Scalar(0, 255, 0), 2, LineTypes.Link4);
}
}
Window.ShowImages(frame);
}
public async Task LoadYoloV2Model()
{
const string cfgFile = @"yolov2.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2.cfg";
const string darknetModel = "yolov2.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov2.weights";
Console.Write("Downloading YoloV2 Model...");
await PrepareFile(cfgFileUrl, cfgFile);
await PrepareFile(darknetModelUrl, darknetModel);
Console.WriteLine(" Done");
using (var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel))
using (var img = Image(@"space_shuttle.jpg"))
{
Assert.False(net.Empty());
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using (var detectionMat = net.Forward("detection_out"))
{
// TODO
GC.KeepAlive(detectionMat);
}
}
}
}
public void NotSupportedUnicodeFileName()
{
Assert.True(File.Exists(LocalModelPath), $"'{LocalModelPath}' not found");
var unicodeFileName = Path.Combine(Path.GetDirectoryName(LocalModelPath) !, "🤣🍀.pb");
if (!File.Exists(unicodeFileName))
{
File.Copy(LocalModelPath, unicodeFileName, true);
}
// Check that ArgumentException(unicode unmappable char) does not occur.
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var ex = Assert.Throws <OpenCVException>(() =>
{
using var net = CvDnn.ReadNet(unicodeFileName);
});
Assert.StartsWith("FAILED: fs.is_open(). Can't open", ex.Message, StringComparison.InvariantCulture);
Assert.Equal("cv::dnn::ReadProtoFromBinaryFile", ex.FuncName);
}
else
{
// No error
}
}
private List <DnnDetectedObject> OptimizeDetections(float threshold, float nmsThreshold, bool nms, List <int> classIds, List <float> confidences, List <float> probabilities, List <Rect2d> boxes)
{
int[] indices;
if (!nms)
{
indices = Enumerable.Range(0, boxes.Count).ToArray();
}
else
{
//using non-maximum suppression to reduce overlapping low confidence box
CvDnn.NMSBoxes(boxes, confidences, threshold, nmsThreshold, out indices);
_logger.LogDebug($"NMSBoxes drop {confidences.Count - indices.Length} overlapping result.");
}
var result = new List <DnnDetectedObject>();
foreach (var i in indices)
{
var box = boxes[i];
var detection = new DnnDetectedObject()
{
Index = classIds[i],
Label = Labels[classIds[i]],
Color = Colors[classIds[i]],
Probability = probabilities[i],
BoundingBox = box
};
result.Add(detection);
}
return(result);
}
public DnnDetectedObject[] ClassifyObjects(Mat image, Rect boxToAnalyze)
{
if (image == null)
{
throw new ArgumentNullException(nameof(image));
}
var blob = CvDnn.BlobFromImage(image, 1.0 / 255, new Size(320, 320), new Scalar(), crop: false);
nnet.SetInput(blob);
//create mats for output layer
Mat[] outs = Enumerable.Repeat(false, _outNames.Length).Select(_ => new Mat()).ToArray();
//forward model
nnet.Forward(outs, _outNames);
const float threshold = 0.5f; //for confidence
const float nmsThreshold = 0.3f; //threshold for nms
var detections = ExtractYolo2Results(outs, image, threshold, nmsThreshold);
blob.Dispose();
foreach (var output in outs)
{
output.Dispose();
}
return(detections);
}
public DnnDetectedObject[][] ClassifyObjects(IEnumerable <Mat> images)
{
if (images is null)
{
throw new ArgumentNullException(nameof(images));
}
var imageList = new List <Mat>(images);
foreach (var image in imageList)
{
if (image?.Empty() == true)
{
throw new ArgumentNullException(nameof(images), "One of the images is not initialized");
}
}
using var blob = CvDnn.BlobFromImages(imageList, 1.0 / 255, new Size(320, 320), crop: false);
nnet.SetInput(blob);
//forward model
nnet.Forward(outs, _outNames);
if (imageList.Count == 1)
{
return(ExtractYolo3SingleResults(outs, imageList[0], threshold, nmsThreshold));
}
else
{
return(ExtractYolo3BatchedResults(outs, images, threshold, nmsThreshold));
}
}
// Start is called before the first frame update
void Start()
{
GameObject textMesh = GameObject.Find("text");
mText = textMesh.GetComponent <TMPro.TextMeshPro>();
mText.SetText("Initializing");
var texture = ScreenCapture.CaptureScreenshotAsTexture();
width = texture.width;
height = texture.height;
stair_width = 0;
stair_height = 0;
stair_x = 0;
stair_y = 0;
threshold = 0.25;
//_texture = new Texture2D((int)(width / scale), (int)(height / scale), TextureFormat.RGBA32, false);
_texture = new Texture2D((1147), (557), TextureFormat.RGBA32, false);
_image = new Mat((557), (1147), MatType.CV_8UC3);
_imageData = new Vec3b[557 * 1147];
//_projectorRenderer.material.mainTexture = _texture;
Object.Destroy(texture);
var m_Path = Application.dataPath;
var cfg = m_Path + "/YOLO/yolo-voc.cfg";
var weights = m_Path + "/YOLO/yolo-voc_best.weights";
net = CvDnn.ReadNetFromDarknet(cfg, weights);
}
public void LoadCaffeModel(String source)
{
Mat img = Cv2.ImRead(source);
using (var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel))
{
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
// Assert.Equal(1, net.GetLayerId(net.GetLayerNames()[0]));
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1.0, new Size(300, 300), new Scalar(104, 117, 123), false, false))
{
net.SetInput(inputBlob, "data");
using (var detection = net.Forward("detection_out"))
{
// find the best class
Console.WriteLine(detection);
Console.WriteLine(detection.Size(2));
GetMaxClass(detection, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} ", classId);
Console.WriteLine("Probability: {0:P2}", classProb);
// Pause();
//Assert.Equal(812, classId);
}
}
}
}
public Rectangle[] Detect(Mat mat, byte[] buffer)
{
var frameWidth = mat.Width;
var frameHeight = mat.Height;
using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(104, 117, 123), false, false);
Net.SetInput(blob);
using var detections = Net.Forward();
using var detectionMat = new Mat(detections.Size(2), detections.Size(3), MatType.CV_32F, detections.Ptr(0));
var rectangles = new List <Rectangle>();
for (var i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
if (confidence > 0.7)
{
var left = (int)(detectionMat.At <float>(i, 3) * frameWidth);
var top = (int)(detectionMat.At <float>(i, 4) * frameHeight);
var right = (int)(detectionMat.At <float>(i, 5) * frameWidth);
var bottom = (int)(detectionMat.At <float>(i, 6) * frameHeight);
rectangles.Add(new Rectangle(left, top, right - left, bottom - top));
}
}
return(rectangles.ToArray());
}
public override void RunTest()
{
const string protoTxt = @"Data\Text\bvlc_googlenet.prototxt";
const string caffeModel = "bvlc_googlenet.caffemodel";
const string synsetWords = @"Data\Text\synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
Console.Write("Downloading Caffe Model...");
PrepareModel(caffeModel);
Console.WriteLine(" Done");
using var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
using var img = new Mat(@"Data\Image\space_shuttle.jpg");
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
// Convert Mat to batch of images
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
net.SetInput(inputBlob, "data");
using var prob = net.Forward("prob");
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
Console.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability: {0:P2}", classProb);
Console.WriteLine("Press any key to exit");
Console.Read();
}
public void LoadCaffeModel()
{
const string protoTxt = @"_data/text/bvlc_googlenet.prototxt";
const string caffeModelUrl = "https://drive.google.com/uc?id=1RUsoiLiXrKBQu9ibwsMqR3n_UkhnZLRR"; //"http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel";
const string caffeModel = "_data/model/bvlc_googlenet.caffemodel";
const string synsetWords = @"_data/text/synset_words.txt";
var classNames = File.ReadAllLines(synsetWords)
.Select(line => line.Split(' ').Last())
.ToArray();
testOutputHelper.WriteLine("Downloading Caffe Model...");
PrepareModel(caffeModelUrl, caffeModel);
testOutputHelper.WriteLine("Done");
using var net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
//Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
var layerName = net.GetLayerNames()[0];
Assert.NotNull(layerName);
Assert.Equal(1, net.GetLayerId(layerName !));
// Convert Mat to batch of images
using var img = Image(@"space_shuttle.jpg");
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
net.SetInput(inputBlob, "data");
using var prob = net.Forward("prob");
// find the best class
GetMaxClass(prob, out int classId, out double classProb);
testOutputHelper.WriteLine("Best class: #{0} '{1}'", classId, classNames[classId]);
testOutputHelper.WriteLine("Probability: {0:P2}", classProb);
Pause();
Assert.Equal(812, classId);
}
//private Net Regogniser { get; set; }
//private Net LabelEncoder { get; set; }
/// <summary>
/// Constructor.
/// </summary>
public DnnCaffeFaceDetector(Double MinConfidence, String detectorPath, string protoFile, string detectorModelFile, string embeddingModelFile,
string recogniserModelFile, string labelEncoderFile) : base(MinConfidence)
{
this.DetectorPath = detectorPath;
this.ProtoFile = protoFile;
this.DetectorModelFile = detectorModelFile;
this.EmbeddingModelFile = embeddingModelFile;
this.RecogniserModelFile = recogniserModelFile;
this.LabelEncoderFile = labelEncoderFile;
try
{
// read in the cafe DNN from disk
this.ProtoPath = System.IO.Path.Join(this.DetectorPath, this.ProtoFile);
this.ModelPath = System.IO.Path.Join(this.DetectorPath, this.DetectorModelFile);
this.Detector = CvDnn.ReadNetFromCaffe(this.ProtoPath, this.ModelPath);
// load the serialised face embedding model from disk.
//this.EmbedderPath = System.IO.Path.Join(this.DetectorPath, this.EmbeddingModelFile);
//this.Embedder = CvDnn.ReadNetFromTorch(this.EmbedderPath);
// todo: load facial recognition model and label encoder.
//this.Regogniser = pickle.loads(open(self._recogniser_model_file, "rb").read())
//this.LabelEncoder = pickle.loads(open(self._label_encoder_file, "rb").read())
}
catch (Exception detail)
{
Globals.Log.Error(detail);
}
}
public Rectangle[] Detect(Mat mat, byte[] buffer)
{
var frameWidth = mat.Width;
var frameHeight = mat.Height;
//using var blob = CvDnn.BlobFromImage(mat, 1.0, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(104, 117, 123), false, false);
using var blob = CvDnn.BlobFromImage(mat, 1.0 / Mean, new OpenCvSharp.Size(300, 300), new OpenCvSharp.Scalar(Mean), true, false);
Net.SetInput(blob);
using var detections = Net.Forward();
using var detectionMat = new Mat(detections.Size(2), detections.Size(3), MatType.CV_32F, detections.Ptr(0));
var bestConfidence = 0.0;
Rectangle?candidate = null;
for (var i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
if ((confidence > 0.5) && (confidence > bestConfidence))
{
var left = (int)(detectionMat.At <float>(i, 3) * frameWidth);
var top = (int)(detectionMat.At <float>(i, 4) * frameHeight);
var right = (int)(detectionMat.At <float>(i, 5) * frameWidth);
var bottom = (int)(detectionMat.At <float>(i, 6) * frameHeight);
candidate = new Rectangle(left, top, right - left, bottom - top);
}
}
if (candidate != null)
{
return(new Rectangle[] { candidate.Value });
}
else
{
return(Array.Empty <Rectangle>());
}
}
public void LoadYoloV2Model()
{
RunGC();
const string cfgFile = @"_data/model/yolov2.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2.cfg";
const string darknetModel = "_data/model/yolov2.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov2.weights";
testOutputHelper.WriteLine("Downloading YoloV2 Model...");
PrepareFile(new Uri(cfgFileUrl), cfgFile);
PrepareFile(new Uri(darknetModelUrl), darknetModel);
testOutputHelper.WriteLine("Done");
RunGC();
using var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel);
Assert.NotNull(net);
Assert.False(net !.Empty());
// Convert Mat to batch of images
using var img = Image(@"space_shuttle.jpg");
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using var detectionMat = net.Forward("detection_out");
// TODO
GC.KeepAlive(detectionMat);
}
protected override void InitializeModel(string pathModel, string pathConfig)
{
//Initialize caffe model
network = CvDnn.ReadNetFromCaffe(pathConfig, pathModel);
this.Scale = 1;
}
private string OpenCVDeepLearningDetector(string path)
{
// uses emugu library
//https://medium.com/@vinuvish/face-detection-with-opencv-and-deep-learning-90bff9028fa8
string prototextPath = @"./Dnn/deploy.prototxt";
string caffeModelPath = @"./Dnn/res10_300x300_ssd_iter_140000.caffemodel";
// load the model;
var net = CvDnn.ReadNetFromCaffe(prototxt: prototextPath, caffeModel: caffeModelPath);
// get the image
OpenCvSharp.Mat image = Cv2.ImRead(path);
// get the original image size
OpenCvSharp.Size imageSize = image.Size();
// the dnn detector works on a 300x300 image;
// now resize the image for the Dnn dector;
OpenCvSharp.Size size = new OpenCvSharp.Size(299, 299);
image = image.Resize(size);
// set the scalar property to RGB colors, don't know what these values represent.
OpenCvSharp.Scalar mcvScalar = new OpenCvSharp.Scalar(104.0, 177.0, 123.0);
var blob = CvDnn.BlobFromImage(image: image, scaleFactor: 1, size: size, mean: mcvScalar, swapRB: true);
net.SetInput(blob);
OpenCvSharp.Mat detections = net.Forward();
// convert the detected values to a faces object that we can use to
// draw rectangles.
List <ConfidenceRect> Faces = new List <ConfidenceRect>();
//var rows = detections.SizeOfDimension[2];
//Array ans = detections.GetData();
//for (int n = 0; n < rows; n++)
//{
// object confidence = ans.GetValue(0, 0, n, 2);
// object x1 = ans.GetValue(0, 0, n, 3);
// object y1 = ans.GetValue(0, 0, n, 4);
// object x2 = ans.GetValue(0, 0, n, 5);
// object y2 = ans.GetValue(0, 0, n, 6);
// ConfidenceRect cr = new ConfidenceRect(confidence, x1, y1, x2, y2, imageSize);
// if (cr.Confidence > 0)
// {
// Debug.WriteLine($"Confidence {cr.Confidence}");
// }
// if (cr.Confidence > Confidence)
// {
// Faces.Add(cr);
// }
//}
//// convert to a writeableBitmap
//WriteableBitmap writeableBitmap = new WriteableBitmap(ImageSource);
//ImageSource = ConvertWriteableBitmapToBitmapImage(writeableBitmap);
//OnPropertyChanged("ImageSource");
//DrawDnnOnImage?.Invoke(Faces, imageSize);
//return Faces.Count.ToString();
return(null);
}
static void Main(string[] args)
{
const string prototext = @"..\..\..\..\data\bvlc_googlenet.prototxt";
const string caffeModel = @"..\..\..\..\data\bvlc_googlenet.caffemodel";
const string synsetWords = @"..\..\..\..\data\synset_words.txt";
string[] classNames = File.ReadAllLines(synsetWords).Select(l => l.Split(' ').Last()).ToArray();
//Use stopwatch object fro timing of the operation
Stopwatch sw = new Stopwatch();
string imgPath = @"D:\DeepLearningOpenCV\images\DogBycleCar.jpg";
using (var net = CvDnn.ReadNetFromCaffe(prototext, caffeModel))
using (var img = Cv2.ImRead(imgPath))
{
//Just out of curiosity, I wanted to get the Layer names of the NN Construct
// by calling GetLayerNames method of the Net object
string[] layerNames = net.GetLayerNames();
Console.WriteLine("Layer names : {0}", string.Join(", ", layerNames));
Console.WriteLine();
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123), swapRB: true, crop: false))
{
sw.Start();
net.SetInput(inputBlob, "data");
using (var prob = net.Forward("prob"))
{
sw.Stop();
Console.WriteLine($"Cost of calculating prob {sw.ElapsedMilliseconds} ms");
int cols = prob.Cols;
int rows = prob.Rows;
Console.WriteLine("Cols: " + cols + ", Rows:" + rows);
// GetMaxProClass(prob, out int classId, out double classProb);
Cv2.MinMaxLoc(prob, out _, out double classProb, out _, out Point classNumberPoint);
int classId = classNumberPoint.X;
Console.WriteLine("Best class: #{0}, '{1}'", classId, classNames[classId]);
Console.WriteLine("Probability:{0:P2}", classProb);
string txt = "Label: " + classNames[classId] + ", % " + (100 * classProb).ToString("0.####");
Cv2.PutText(img, txt, new Point(5, 25), HersheyFonts.HersheySimplex, 0.7, new Scalar(0, 0, 255), 2);
//Cv2.ImWrite("classification.jpg", img);
Cv2.ImShow("image", img);
}
}
}
Cv2.WaitKey();
Cv2.DestroyAllWindows();
// Console.Write("Downloading Caffe Model...");
Console.WriteLine("Press any key to exit");
Console.Read();
}
public CaffeDnnFaceDetector(string modelFile)
{
var modelDirectory = Path.GetDirectoryName(modelFile) !;
var configFile = Path.Combine(modelDirectory, "deploy.prototxt");
Net = CvDnn.ReadNetFromCaffe(configFile, modelFile);
Net.SetPreferableBackend(Net.Backend.INFERENCE_ENGINE);
Net.SetPreferableTarget(Net.Target.OPENCL_FP16);
}
protected override Mat GetFaces(Mat frame, out FacesList facesList)
{
// debug - replace video stream with still image for testing
//var file = "C:\\Users\\Lamby\\Documents\\Visual Studio 2017\\Projects\\TrackingCamera\\TrackingCamera\\DetectorClasses\\ModelDetectorVGG_VOC0712Plus\\bali-crop.jpg";
string file = "C:\\Users\\Lamby\\Desktop\\fd-acc-result3-e1539872783684.jpg";
frame = Cv2.ImRead(file);
Mat imageBlob = CvDnn.BlobFromImage(frame, 1.0, new Size(300, 300),
new Scalar(104.0, 177.0, 123.0), false, false);
this.Detector.SetInput(imageBlob, "data");
Mat detections = this.Detector.Forward("detection_out");
//reshape from [1,1,200,7] to [200,7]
Mat detectionMat = detections.Reshape(1, detections.Size(2));
// debug
//GetFaceBestConfidence(detections, out int faceId, out double faceProbability);
if (detectionMat.Rows <= 0) //
{
facesList = new FacesList();
return(null);
}
else
{
facesList = new FacesList();
Scalar rgbColour = new Scalar(0, 255, 255);
for (int i = 0; i < detectionMat.Rows; i++)
{
var confidence = detectionMat.At <float>(i, 2);
if (confidence > this.MinConfidence)
{
int X1 = (int)(detectionMat.At <float>(i, 3) * frame.Width); //detectionMat.At<int> returns 0 with this floating point caffe model?
int Y1 = (int)(detectionMat.At <float>(i, 4) * frame.Height);
int X2 = (int)(detectionMat.At <float>(i, 5) * frame.Width);
int Y2 = (int)(detectionMat.At <float>(i, 6) * frame.Height);
frame.Rectangle(new Point(X1, Y1), new Point(X2, Y2), rgbColour, 2, OpenCvSharp.LineTypes.Link4);
string faceText = String.Format("{0:P2}", confidence);
Cv2.PutText(frame, faceText, new Point(X1, Y2 + 9), HersheyFonts.HersheyComplex, 0.3, rgbColour);
var faceMat = frame[new Rect(X1, Y1, X2 - X1, Y2 - Y1)];
facesList.Add(new Face(faceMat, new Point(X1, Y1), new Point(X2, Y2), confidence));
// Debug
//Cv2.ImShow("Detected Face", faceMat);
//Cv2.WaitKey(1);
}
}
return(frame);
}
}
public TensorFlowDnnFaceDetector(string modelFile)
{
var modelDirectory = Path.GetDirectoryName(modelFile) !;
var configFile = Path.Combine(modelDirectory, "opencv_face_detector.pbtxt");
Net = CvDnn.ReadNetFromTensorflow(modelFile, configFile);
// Net.SetPreferableTarget(Net.Target.OPENCL_FP16);
Net.SetPreferableBackend(Net.Backend.OPENCV);
}
public void DetectAllText(string fileName)
{
const int InputWidth = 320;
const int InputHeight = 320;
const float ConfThreshold = 0.5f;
const float NmsThreshold = 0.4f;
// Load network.
using (Net net = CvDnn.ReadNet(Path.GetFullPath(LocalModelPath)))
using (Mat img = new Mat(fileName))
// Prepare input image
using (var blob = CvDnn.BlobFromImage(img, 1.0, new Size(InputWidth, InputHeight), new Scalar(123.68, 116.78, 103.94), true, false))
{
// Forward Pass
// Now that we have prepared the input, we will pass it through the network. There are two outputs of the network.
// One specifies the geometry of the Text-box and the other specifies the confidence score of the detected box.
// These are given by the layers :
// feature_fusion/concat_3
// feature_fusion/Conv_7/Sigmoid
var outputBlobNames = new string[] { "feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3" };
var outputBlobs = outputBlobNames.Select(_ => new Mat()).ToArray();
net.SetInput(blob);
net.Forward(outputBlobs, outputBlobNames);
Mat scores = outputBlobs[0];
Mat geometry = outputBlobs[1];
// Decode predicted bounding boxes (decode the positions of the text boxes along with their orientation)
Decode(scores, geometry, ConfThreshold, out var boxes, out var confidences);
// Apply non-maximum suppression procedure for filtering out the false positives and get the final predictions
CvDnn.NMSBoxes(boxes, confidences, ConfThreshold, NmsThreshold, out var indices);
// Render detections.
Point2f ratio = new Point2f((float)img.Cols / InputWidth, (float)img.Rows / InputHeight);
for (var i = 0; i < indices.Length; ++i)
{
RotatedRect box = boxes[indices[i]];
Point2f[] vertices = box.Points();
for (int j = 0; j < 4; ++j)
{
vertices[j].X *= ratio.X;
vertices[j].Y *= ratio.Y;
}
for (int j = 0; j < 4; ++j)
{
Cv2.Line(img, (int)vertices[j].X, (int)vertices[j].Y, (int)vertices[(j + 1) % 4].X, (int)vertices[(j + 1) % 4].Y, new Scalar(0, 255, 0), 3);
}
}
ShowImagesWhenDebugMode(img);
}
}
public void initialize(string protoTxt, string caffeModel, string synsetWords)
{
classNames = File.ReadAllLines(synsetWords).Select(line => line.Split(' ').Last()).ToArray();
PrepareModel(caffeModel);
net = CvDnn.ReadNetFromCaffe(protoTxt, caffeModel);
Console.WriteLine("Layer names: {0}", string.Join(", ", net.GetLayerNames()));
Console.WriteLine();
Console.WriteLine("Preparation complete");
}
public InferenceEngineDetector(string modelFile)
{
Net = CvDnn.ReadNet(Path.ChangeExtension(modelFile, ".bin"), Path.ChangeExtension(modelFile, ".xml"));
Net.SetPreferableBackend(Net.Backend.INFERENCE_ENGINE);
Net.SetPreferableTarget(Net.Target.OPENCL_FP16);
//Net.SetPreferableTarget(Net.Target.OPENCL_FP16);
//Net.SetPreferableBackend(Net.Backend.OPENCV);
OutputMat = new Mat();
}
public TinyYoloV3Detector(string modelFile)
{
Net = CvDnn.ReadNetFromDarknet(Path.ChangeExtension(modelFile, ".cfg"), Path.ChangeExtension(modelFile, ".weights"));
//Net.SetPreferableBackend(Net.Backend.INFERENCE_ENGINE);
Net.SetPreferableBackend(Net.Backend.OPENCV);
Net.SetPreferableTarget(Net.Target.CPU);
OutputLayerIds = Net.GetUnconnectedOutLayers();
LayerNames = Net.GetLayerNames();
OutputLayerNames = Net.GetUnconnectedOutLayersNames() !;
}
public void LoadYoloV3Model()
{
RunGC();
const string cfgFile = @"_data/model/yolov3.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg";
const string darknetModel = "_data/model/yolov3.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov3.weights";
testOutputHelper.WriteLine("Downloading YoloV3 Model...");
PrepareFile(new Uri(cfgFileUrl), cfgFile);
PrepareFile(new Uri(darknetModelUrl), darknetModel);
testOutputHelper.WriteLine("Done");
RunGC();
using (var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel))
using (var img = Image(@"space_shuttle.jpg"))
{
Assert.NotNull(net);
Assert.False(net !.Empty());
var outNames = net.GetUnconnectedOutLayersNames();
Assert.NotEmpty(outNames);
Assert.DoesNotContain(outNames, elem => elem == null);
testOutputHelper.WriteLine("UnconnectedOutLayersNames: {0}", string.Join(",", outNames));
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using (var detection82 = net.Forward("yolo_82"))
using (var detection94 = net.Forward("yolo_94"))
using (var detection106 = net.Forward("yolo_106"))
{
// TODO
Assert.False(detection82.Empty());
Assert.False(detection94.Empty());
Assert.False(detection106.Empty());
}
Mat[] outs = outNames.Select(_ => new Mat()).ToArray();
net.Forward(outs, outNames !);
foreach (var m in outs)
{
Assert.False(m.Empty());
m.Dispose();
}
}
}
}
//Method
public void InitializeDetector(string cfg, string names, string weights, Size blobSize, float confThresh, float nmsThresh)
{
BlobSize = blobSize;
_threshold = confThresh;
_nmsThreshold = nmsThresh;
_labels = File.ReadAllLines(names).ToArray();
_net = CvDnn.ReadNetFromDarknet(cfg, weights);
_net.SetPreferableBackend(Net.Backend.OPENCV);
_net.SetPreferableTarget(Net.Target.CPU);
_status = Status.Initialized;
}
public CaffeDnnFaceDetector(string modelFile)
{
var configFile = Path.ChangeExtension(modelFile, ".prototxt") !;
Net = CvDnn.ReadNetFromCaffe(configFile, modelFile);
//Cv2.SetNumThreads(2);
//Net.SetPreferableBackend(Net.Backend.OPENCV);
Net.SetPreferableBackend(Net.Backend.INFERENCE_ENGINE);
Net.SetPreferableTarget(Net.Target.MYRIAD);
//Net.SetPreferableTarget(Net.Target.OPENCL);
}
private void bWorker_DoWork(object sender, DoWorkEventArgs e)
{
//Loading YOLO config
Darknet darknet = new Darknet(YoloCfgFile, YoloWeightsFile);
var classNames = File.ReadAllLines(YoloNamesFile);
//Capturing frames
for (; ;)
{
Mat originalBackground = this.background.Clone();
Mat frame;
frame = cam.RetrieveMat();
if (frame.Empty())
{
Cv2.WaitKey();
return;
}
Mat inputBlob = CvDnn.BlobFromImage(frame, 1 / 255d, new OpenCvSharp.Size(544, 544), new Scalar(), true, false);
var results = darknet.Detect(frame, 0.5f);
foreach (var result in results)
{
var p1 = new OpenCvSharp.Point(result.x, result.y);
var p2 = new OpenCvSharp.Point(result.x + result.w, result.y + result.h);
if (CheckContoursAmount(contours, binary, p1, p2))
{
//Formatting labels
var label = $"{classNames[result.obj_id]}";
var textSize = Cv2.GetTextSize(label,
HersheyFonts.HersheyTriplex,
0.5,
1,
out var baseline);
Cv2.Rectangle(originalBackground, p1, p2, Colors[result.obj_id], -1);
Cv2.PutText(
originalBackground,
label,
new OpenCvSharp.Point((int)(result.x + (result.w / 2.0) - (textSize.Width / 2.0)), (int)(result.y + (result.h / 2.0))),
HersheyFonts.HersheyTriplex,
0.5,
Scalar.Black);
}
}
bWorker.ReportProgress(0, originalBackground);
}
}
static void Main()
{
var file = "bali.jpg";
var prototxt = "deploy.prototxt";
var model = "VGG_VOC0712Plus_SSD_512x512_ft_iter_160000.caffemodel";
var colors = Enumerable.Repeat(false, 21).Select(x => Scalar.RandomColor()).ToArray();
//get image
var org = Cv2.ImRead(file);
var blob = CvDnn.BlobFromImage(org, 1, new Size(512, 512));
//setup model
var net = CvDnn.ReadNetFromCaffe(prototxt, model);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward("detection_out");
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
//reshape from [1,1,200,7] to [200,7]
var p = prob.Reshape(1, prob.Size(2));
for (int i = 0; i < prob.Size(2); i++)
{
var confidence = p.At <float>(i, 2);
if (confidence > 0.4)
{
//get value what we need
var idx = (int)p.At <float>(i, 1);
var w1 = (int)(org.Width * p.At <float>(i, 3));
var h1 = (int)(org.Width * p.At <float>(i, 4));
var w2 = (int)(org.Width * p.At <float>(i, 5));
var h2 = (int)(org.Width * p.At <float>(i, 6));
var label = $"{Labels[idx]} {confidence * 100:0.00}%";
Console.WriteLine($"{label}");
//draw result
Cv2.Rectangle(org, new Rect(w1, h1, w2 - w1, h2 - h1), colors[idx], 2);
var textSize = Cv2.GetTextSize(label, HersheyFonts.HersheyTriplex, 0.5, 1, out var baseline);
Cv2.Rectangle(org, new Rect(new Point(w1, h1 - textSize.Height),
new Size(textSize.Width, textSize.Height + baseline)), colors[idx], Cv2.FILLED);
Cv2.PutText(org, label, new Point(w1, h1), HersheyFonts.HersheyTriplex, 0.5, Scalar.Black);
}
}
using (new Window("image", org))
{
Cv2.WaitKey();
}
}
