public async Task LoadYoloV2Model()
{
const string cfgFile = @"yolov2.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2.cfg";
const string darknetModel = "yolov2.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov2.weights";
Console.Write("Downloading YoloV2 Model...");
await PrepareFile(cfgFileUrl, cfgFile);
await PrepareFile(darknetModelUrl, darknetModel);
Console.WriteLine(" Done");
using (var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel))
using (var img = Image(@"space_shuttle.jpg"))
{
Assert.False(net.Empty());
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using (var detectionMat = net.Forward("detection_out"))
{
// TODO
GC.KeepAlive(detectionMat);
}
}
}
}
// Start is called before the first frame update
void Start()
{
GameObject textMesh = GameObject.Find("text");
mText = textMesh.GetComponent <TMPro.TextMeshPro>();
mText.SetText("Initializing");
var texture = ScreenCapture.CaptureScreenshotAsTexture();
width = texture.width;
height = texture.height;
stair_width = 0;
stair_height = 0;
stair_x = 0;
stair_y = 0;
threshold = 0.25;
//_texture = new Texture2D((int)(width / scale), (int)(height / scale), TextureFormat.RGBA32, false);
_texture = new Texture2D((1147), (557), TextureFormat.RGBA32, false);
_image = new Mat((557), (1147), MatType.CV_8UC3);
_imageData = new Vec3b[557 * 1147];
//_projectorRenderer.material.mainTexture = _texture;
Object.Destroy(texture);
var m_Path = Application.dataPath;
var cfg = m_Path + "/YOLO/yolo-voc.cfg";
var weights = m_Path + "/YOLO/yolo-voc_best.weights";
net = CvDnn.ReadNetFromDarknet(cfg, weights);
}
public void LoadYoloV2Model()
{
RunGC();
const string cfgFile = @"_data/model/yolov2.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2.cfg";
const string darknetModel = "_data/model/yolov2.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov2.weights";
testOutputHelper.WriteLine("Downloading YoloV2 Model...");
PrepareFile(new Uri(cfgFileUrl), cfgFile);
PrepareFile(new Uri(darknetModelUrl), darknetModel);
testOutputHelper.WriteLine("Done");
RunGC();
using var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel);
Assert.NotNull(net);
Assert.False(net !.Empty());
// Convert Mat to batch of images
using var img = Image(@"space_shuttle.jpg");
using var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123));
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using var detectionMat = net.Forward("detection_out");
// TODO
GC.KeepAlive(detectionMat);
}
public TinyYoloV3Detector(string modelFile)
{
Net = CvDnn.ReadNetFromDarknet(Path.ChangeExtension(modelFile, ".cfg"), Path.ChangeExtension(modelFile, ".weights"));
//Net.SetPreferableBackend(Net.Backend.INFERENCE_ENGINE);
Net.SetPreferableBackend(Net.Backend.OPENCV);
Net.SetPreferableTarget(Net.Target.CPU);
OutputLayerIds = Net.GetUnconnectedOutLayers();
LayerNames = Net.GetLayerNames();
OutputLayerNames = Net.GetUnconnectedOutLayersNames() !;
}
public void LoadYoloV3Model()
{
RunGC();
const string cfgFile = @"_data/model/yolov3.cfg";
const string cfgFileUrl = "https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg";
const string darknetModel = "_data/model/yolov3.weights";
const string darknetModelUrl = "https://pjreddie.com/media/files/yolov3.weights";
testOutputHelper.WriteLine("Downloading YoloV3 Model...");
PrepareFile(new Uri(cfgFileUrl), cfgFile);
PrepareFile(new Uri(darknetModelUrl), darknetModel);
testOutputHelper.WriteLine("Done");
RunGC();
using (var net = CvDnn.ReadNetFromDarknet(cfgFile, darknetModel))
using (var img = Image(@"space_shuttle.jpg"))
{
Assert.NotNull(net);
Assert.False(net !.Empty());
var outNames = net.GetUnconnectedOutLayersNames();
Assert.NotEmpty(outNames);
Assert.DoesNotContain(outNames, elem => elem == null);
testOutputHelper.WriteLine("UnconnectedOutLayersNames: {0}", string.Join(",", outNames));
// Convert Mat to batch of images
using (var inputBlob = CvDnn.BlobFromImage(img, 1, new Size(224, 224), new Scalar(104, 117, 123)))
{
// Set input blob
net.SetInput(inputBlob, "data");
// Make forward pass
using (var detection82 = net.Forward("yolo_82"))
using (var detection94 = net.Forward("yolo_94"))
using (var detection106 = net.Forward("yolo_106"))
{
// TODO
Assert.False(detection82.Empty());
Assert.False(detection94.Empty());
Assert.False(detection106.Empty());
}
Mat[] outs = outNames.Select(_ => new Mat()).ToArray();
net.Forward(outs, outNames !);
foreach (var m in outs)
{
Assert.False(m.Empty());
m.Dispose();
}
}
}
}
//Method
public void InitializeDetector(string cfg, string names, string weights, Size blobSize, float confThresh, float nmsThresh)
{
BlobSize = blobSize;
_threshold = confThresh;
_nmsThreshold = nmsThresh;
_labels = File.ReadAllLines(names).ToArray();
_net = CvDnn.ReadNetFromDarknet(cfg, weights);
_net.SetPreferableBackend(Net.Backend.OPENCV);
_net.SetPreferableTarget(Net.Target.CPU);
_status = Status.Initialized;
}
public YoloV3Detector()
{
labels = File.ReadAllLines(YoloNames).ToArray();
net = CvDnn.ReadNetFromDarknet(YoloConfig, YoloWeights);
net.SetPreferableBackend(Backend.CUDA);
net.SetPreferableTarget(Target.CUDA);
outputNames = net.GetUnconnectedOutLayersNames();
outputLayers = outputNames.Select(_ => new Mat()).ToArray();
var configs = File.ReadAllLines(YoloConfig).ToArray();
var configWidth = configs.Where(x => x.StartsWith("width="))
.Select(x => int.Parse(x.Split('=')[1]))
.FirstOrDefault();
var configHeight = configs.Where(x => x.StartsWith("height="))
.Select(x => int.Parse(x.Split('=')[1]))
.FirstOrDefault();
configSize = new Size(configWidth, configHeight);
blobFromImageMeanParams = new Scalar();
scalarFactor = 1.0 / 255;
}
static void Main(string[] args)
{
Console.WriteLine("Trying to capture video yolov4");
#region parameter
var image = Path.Combine(Location, "kite.jpg");
var cfg = Path.Combine(Location, Cfg);
var model = Path.Combine(Location, Weight);
const float threshold = 0.5f; //for confidence
const float nmsThreshold = 0.3f; //threshold for nms
#endregion
using var capture = new VideoCapture(0);
int sleepTime = (int)Math.Round(1000 / capture.Fps);
using (var window = new Window("capture"))
{
var org = new Mat();
//load model and config, if you got error: "separator_index < line.size()", check your cfg file, must be something wrong.
var net = CvDnn.ReadNetFromDarknet(cfg, model);
#region set preferable
net.SetPreferableBackend(Net.Backend.OPENCV);
/*
* 0:DNN_BACKEND_DEFAULT
* 1:DNN_BACKEND_HALIDE
* 2:DNN_BACKEND_INFERENCE_ENGINE
* 3:DNN_BACKEND_OPENCV
*/
net.SetPreferableTarget(0);
/*
* 0:DNN_TARGET_CPU
* 1:DNN_TARGET_OPENCL
* 2:DNN_TARGET_OPENCL_FP16
* 3:DNN_TARGET_MYRIAD
* 4:DNN_TARGET_FPGA
*/
#endregion
while (true)
{
//get image
capture.Read(org);
//setting blob, size can be:320/416/608
//opencv blob setting can check here https://github.com/opencv/opencv/tree/master/samples/dnn#object-detection
var blob = CvDnn.BlobFromImage(org, 1.0 / 255, new Size(320, 320), new Scalar(), true, false);
//input data
net.SetInput(blob);
//get output layer name
var outNames = net.GetUnconnectedOutLayersNames();
//create mats for output layer
var outs = outNames.Select(_ => new Mat()).ToArray();
#region forward model
Stopwatch sw = new Stopwatch();
sw.Start();
net.Forward(outs, outNames);
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
#endregion
//get result from all output
GetResult(outs, org, threshold, nmsThreshold, true);
//using (new Window("died.tw", org))
//{
// Cv2.WaitKey();
//}
window.ShowImage(org);
Cv2.WaitKey(sleepTime);
}
}
}
public static BindableCollection <YoloModel> GetYoloModels()
{
const string cocoPath = "dataset/coco";
const string vocPath = "dataset/voc";
const string tiny = "tiny";
try
{
var cocoNames = File.ReadAllLines($"{cocoPath}.names");
var vocNames = File.ReadAllLines($"{vocPath}.names");
var cocoColors = Enumerable.Repeat(false, cocoNames.Length).Select(x => Scalar.RandomColor()).ToArray();
var vocColors = Enumerable.Repeat(false, vocNames.Length).Select(x => Scalar.RandomColor()).ToArray();
// weights
// coco.weights https://pjreddie.com/media/files/yolov2.weights
// cocotiny.weights https://pjreddie.com/media/files/yolov2-tiny.weights
// voc.weights https://pjreddie.com/media/files/yolov2-voc.weights
// voctiny.weights https://pjreddie.com/media/files/yolov2-tiny-voc.weights
var yoloModels = new BindableCollection <YoloModel>
{
new YoloModel
{
Name = "Coco",
_cfg = $"{cocoPath}.cfg",
_labels = cocoNames,
_colors = cocoColors,
_net = CvDnn.ReadNetFromDarknet($"{cocoPath}.cfg", $"{cocoPath}.weights")
},
new YoloModel
{
Name = "Tiny Coco",
_cfg = $"{cocoPath}{tiny}.cfg",
_labels = cocoNames,
_colors = cocoColors,
_net = CvDnn.ReadNetFromDarknet($"{cocoPath}{tiny}.cfg", $"{cocoPath}{tiny}.weights")
},
new YoloModel
{
Name = "Voc",
_cfg = $"{vocPath}.cfg",
_labels = vocNames,
_colors = vocColors,
_net = CvDnn.ReadNetFromDarknet($"{vocPath}.cfg", $"{vocPath}.weights")
},
new YoloModel
{
Name = "Tiny Voc",
_cfg = $"{vocPath}{tiny}.cfg",
_labels = vocNames,
_colors = vocColors,
_net = CvDnn.ReadNetFromDarknet($"{vocPath}{tiny}.cfg", $"{vocPath}{tiny}.weights")
}
};
return(yoloModels);
}
catch (Exception ex)
{
UiServices.ShowError(ex);
System.Windows.Application.Current.Shutdown();
}
return(null);
}
static void Main()
{
var file = "bali.jpg";
// https://pjreddie.com/darknet/yolo/
var cfg = "yolo-voc.cfg";
var model = "yolo-voc.weights"; //YOLOv2 544x544
var threshold = 0.3;
var org = Cv2.ImRead(file);
var w = org.Width;
var h = org.Height;
//setting blob, parameter are important
var blob = CvDnn.BlobFromImage(org, 1 / 255.0, new Size(544, 544), new Scalar(), true, false);
var net = CvDnn.ReadNetFromDarknet(cfg, model);
net.SetInput(blob, "data");
Stopwatch sw = new Stopwatch();
sw.Start();
//forward model
var prob = net.Forward();
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
/* YOLO2 VOC output
* 0 1 : center 2 3 : w/h
* 4 : confidence 5 ~24 : class probability */
const int prefix = 5; //skip 0~4
for (int i = 0; i < prob.Rows; i++)
{
var confidence = prob.At <float>(i, 4);
if (confidence > threshold)
{
//get classes probability
Cv2.MinMaxLoc(prob.Row[i].ColRange(prefix, prob.Cols), out _, out Point max);
var classes = max.X;
var probability = prob.At <float>(i, classes + prefix);
if (probability > threshold) //more accuracy
{
//get center and width/height
var centerX = prob.At <float>(i, 0) * w;
var centerY = prob.At <float>(i, 1) * h;
var width = prob.At <float>(i, 2) * w;
var height = prob.At <float>(i, 3) * h;
//label formating
var label = $"{Labels[classes]} {probability * 100:0.00}%";
Console.WriteLine($"confidence {confidence * 100:0.00}% {label}");
var x1 = (centerX - width / 2) < 0 ? 0 : centerX - width / 2; //avoid left side over edge
//draw result
org.Rectangle(new Point(x1, centerY - height / 2), new Point(centerX + width / 2, centerY + height / 2), Colors[classes], 2);
var textSize = Cv2.GetTextSize(label, HersheyFonts.HersheyTriplex, 0.5, 1, out var baseline);
Cv2.Rectangle(org, new Rect(new Point(x1, centerY - height / 2 - textSize.Height - baseline),
new Size(textSize.Width, textSize.Height + baseline)), Colors[classes], Cv2.FILLED);
Cv2.PutText(org, label, new Point(x1, centerY - height / 2 - baseline), HersheyFonts.HersheyTriplex, 0.5, Scalar.Black);
}
}
}
using (new Window("died.tw", org))
{
Cv2.WaitKey();
}
}
private static string model = "yolov2-voc.weights"; //YOLOv2 544x544
public static YOLODescriptor Detect(Mat image, float threshold, bool enabled)
{
SpreadBuilder <Rect> rectSB = Spread.CreateBuilder <Rect>();
SpreadBuilder <float> confidenceSB = Spread.CreateBuilder <float>();
SpreadBuilder <int> detectedClassSB = Spread.CreateBuilder <int>();
SpreadBuilder <float> classProbabilitySB = Spread.CreateBuilder <float>();
SpreadBuilder <string> labelSB = Spread.CreateBuilder <string>();
if (enabled && image != DefaultMat.Damon)
{
var w = image.Width;
var h = image.Height;
//setting blob, parameter are important
var blob = CvDnn.BlobFromImage(image, 1 / 255.0, new Size(544, 544), new Scalar(), true, false);
if (net == null || net.IsDisposed)
{
net = CvDnn.ReadNetFromDarknet(cfg, model);
}
net.SetInput(blob, "data");
//forward model
var prob = net.Forward();
/* YOLO2 VOC output
* 0 1 : center 2 3 : w/h
* 4 : confidence 5 ~24 : class probability */
const int prefix = 5; //skip 0~4
for (int i = 0; i < prob.Rows; i++)
{
var matchConfidence = prob.At <float>(i, 4);
if (matchConfidence > threshold)
{
//get classes probability
Point min, max;
Cv2.MinMaxLoc(prob.Row[i].ColRange(prefix, prob.Cols), out min, out max);
var classes = max.X;
var probability = prob.At <float>(i, classes + prefix);
if (probability > threshold) //more accuracy
{
//get center and width/height
var centerX = prob.At <float>(i, 0) * w;
var centerY = prob.At <float>(i, 1) * h;
var width = prob.At <float>(i, 2) * w;
var height = prob.At <float>(i, 3) * h;
var x1 = (centerX - width / 2) < 0 ? 0 : centerX - width / 2; //avoid left side over edge
//org.Rectangle(new Point(centerX - width / 2, centerY - height / 2), new Point(centerX + width / 2, centerY + height / 2), Colors[classes], 2);
rectSB.Add(new Rect(new Point(x1, centerY - height / 2), new Size(width, height)));
confidenceSB.Add(matchConfidence);
detectedClassSB.Add(classes);
classProbabilitySB.Add(probability);
labelSB.Add(Labels[classes]);
}
}
}
}
YOLODescriptor result = new YOLODescriptor();
result.confidence = confidenceSB.ToSpread();
result.detectedClass = detectedClassSB.ToSpread();
result.classProbability = classProbabilitySB.ToSpread();
result.classLabel = labelSB.ToSpread();
result.rectangles = rectSB.ToSpread <Rect>();
return(result);
}
static void Main()
{
#region parameter
Console.Write("Enter name and suffix of image in content folder, e.g 1.png, : ");
var img = @Console.ReadLine();
var image = Path.Combine(Location, img);
var cfg = Path.Combine(Location, Cfg);
var model = Path.Combine(Location, Weight);
const float threshold = 0.3f; //for confidence
const float nmsThreshold = 0.3f; //threshold for nms
#endregion
//get image
var org = new Mat(image);
//setting blob, size can be:320/416/608
//opencv blob setting can check here https://github.com/opencv/opencv/tree/master/samples/dnn#object-detection
var blob = CvDnn.BlobFromImage(org, 1.0 / 255, new Size(416, 416), new Scalar(), true, false);
//load model and config, if you got error: "separator_index < line.size()", check your cfg file, must be something wrong.
var net = CvDnn.ReadNetFromDarknet(cfg, model);
#region set preferable
net.SetPreferableBackend(3);
/*
* 0:DNN_BACKEND_DEFAULT
* 1:DNN_BACKEND_HALIDE
* 2:DNN_BACKEND_INFERENCE_ENGINE
* 3:DNN_BACKEND_OPENCV
*/
net.SetPreferableTarget(0);
/*
* 0:DNN_TARGET_CPU
* 1:DNN_TARGET_OPENCL
* 2:DNN_TARGET_OPENCL_FP16
* 3:DNN_TARGET_MYRIAD
* 4:DNN_TARGET_FPGA
*/
#endregion
//input data
net.SetInput(blob);
//get output layer name
var outNames = net.GetUnconnectedOutLayersNames();
//create mats for output layer
var outs = outNames.Select(_ => new Mat()).ToArray();
#region forward model
Stopwatch sw = new Stopwatch();
sw.Start();
net.Forward(outs, outNames);
sw.Stop();
Console.WriteLine($"Runtime:{sw.ElapsedMilliseconds} ms");
#endregion
//get result from all output
GetResult(outs, org, threshold, nmsThreshold);
Cv2.ImWrite(Location + "Resultfor_" + img, org);
using (new Window("Resultfor_" + img, org))
{
Cv2.WaitKey();
}
}
static void Main(string[] args)
{
// https://pjreddie.com/darknet/yolo/
//var cfg = "yolo-voc.cfg";
//var model = "yolo-voc.weights"; //YOLOv2 544x544
//var size = new Size(544, 544);
//var cfg = "yolov3.cfg";
//var model = "yolov3.weights"; //YOLOv2 544x544
//var size = new Size(608, 608);
//var cfg = "yolov3-tiny.cfg";
//var model = "yolov3-tiny.weights";
//var size = new Size(416, 416);
var cfg = "yolov2.cfg";
var model = "yolov2.weights";
var net = CvDnn.ReadNetFromDarknet(cfg, model);
string file;
if (args.Length < 1)
{
//file = "bali.jpg";
file = "dog.jpg";
Process(net, file, true);
}
else if (args.Length == 1)
{
// 引数が画像1つの場合はウィンドウで画像表示
file = args[0];
LogError(file);
if (File.Exists(file))
{
Process(net, file, true);
}
else if (Directory.Exists(file))
{
// フォルダならばその中を一括で処理
ProcessDirectory(net, file);
//Log("");
//Log("何かキーを押すと終了します");
//Console.ReadKey();
}
}
else
{
// 複数の引数があった場合はウィンドウ表示はなし
foreach (var path in args)
{
if (File.Exists(path))
{
Process(net, path, false);
}
else
{
ProcessDirectory(net, path);
}
}
//Log("");
//Log("何かキーを押すと終了します");
//Console.ReadKey();
}
}