private static void RandomlyCropImage(Matrix <RgbPixel> inputImage, Matrix <ushort> labelImage, TrainingSample crop, Rand rnd)
{
var rect = MakeRandomCroppingRectResNet(inputImage, rnd);
using (var chipDims = new ChipDims(227, 227))
using (var chipDetails = new ChipDetails(rect, chipDims))
{
// Crop the input image.
crop.InputImage = Dlib.ExtractImageChip <RgbPixel>(inputImage, chipDetails, InterpolationTypes.Bilinear);
// Crop the labels correspondingly. However, note that here bilinear
// interpolation would make absolutely no sense - you wouldn't say that
// a bicycle is half-way between an aeroplane and a bird, would you?
crop.LabelImage = Dlib.ExtractImageChip <ushort>(labelImage, chipDetails, InterpolationTypes.NearestNeighbor);
// Also randomly flip the input image and the labels.
if (rnd.GetRandomDouble() > 0.5)
{
var tmpInput = Dlib.FlipLR(crop.InputImage);
var tmpLabel = Dlib.FlipLR(crop.LabelImage);
crop.InputImage?.Dispose();
crop.LabelImage?.Dispose();
crop.InputImage = tmpInput;
crop.LabelImage = tmpLabel;
}
// And then randomly adjust the colors.
Dlib.ApplyRandomColorOffset(crop.InputImage, rnd);
}
}
public void ExtractImageChip()
{
var path = this.GetDataFile($"{LoadTarget}.bmp");
const int loop = 1000;
var sizeArray = new long[loop];
var first = GetCurrentMemory();
var start = GetCurrentMemory() - first;
using (var image = DlibTest.LoadImage(ImageTypes.RgbPixel, path))
using (var dims = new ChipDims(227, 227))
using (var chip = new ChipDetails(new Rectangle(0, 0, 100, 100), dims))
for (var count = 0; count < loop; count++)
{
using (Dlib.ExtractImageChip <RgbPixel>(image, chip))
sizeArray[count] = GetCurrentMemory();
}
// Important!!
GC.Collect(2, GCCollectionMode.Forced, true);
var end = GetCurrentMemory() - first;
Console.WriteLine(" Start Total Memory = {0} KB", start / 1024);
Console.WriteLine(" End Total Memory = {0} KB", end / 1024);
Console.WriteLine("Delta (End - Start) Memory = {0} KB", (end - start) / 1024);
// Rough estimate whether occur memory leak (less than 10240KB)
Assert.IsTrue((end - start) / 1024 < 10240);
}
private void buscarrosto(Bitmap frame)
{
Image <Rgb, Byte> imageCV = new Image <Rgb, byte>(frame);
Emgu.CV.Mat mat = imageCV.Mat;
var array = new byte[mat.Width * mat.Height * mat.ElementSize];
mat.CopyTo(array);
using (Array2D <RgbPixel> image = Dlib.LoadImageData <RgbPixel>(array, (uint)mat.Height, (uint)mat.Width, (uint)(mat.Width * mat.ElementSize)))
{
using (FrontalFaceDetector fd = Dlib.GetFrontalFaceDetector())
{
var faces = fd.Operator(image);
foreach (DlibDotNet.Rectangle face in faces)
{
FullObjectDetection shape = _ShapePredictor.Detect(image, face);
ChipDetails faceChipDetail = Dlib.GetFaceChipDetails(shape, 150, 0.25);
Array2D <RgbPixel> faceChip = Dlib.ExtractImageChip <RgbPixel>(image, faceChipDetail);
Bitmap bitmap1 = faceChip.ToBitmap <RgbPixel>();
MainWindow.main.Statusa1 = bitmap1;
Dlib.DrawRectangle(image, face, color: new RgbPixel(0, 255, 255), thickness: 4);
}
}
frame = image.ToBitmap <RgbPixel>();
MainWindow.main.Statusa = frame;
}
}
public void CopyTo()
{
const int size = 10;
var source = Enumerable.Range(0, size).Select(i => new ChipDetails(new DRectangle(i, i, i, i), (uint)i));
var vector = new StdVector <ChipDetails>(source);
Assert.Equal(vector.Size, size);
var ret = new ChipDetails[15];
vector.CopyTo(ret, 5);
for (var i = 0; i < size; i++)
{
Assert.Equal(ret[i + 5].Rect.Left, i);
Assert.Equal(ret[i + 5].Rect.Top, i);
Assert.Equal(ret[i + 5].Rect.Right, i);
Assert.Equal(ret[i + 5].Rect.Bottom, i);
}
this.DisposeAndCheckDisposedState(vector);
}
public void GetImage(string imagePath)
{
Array2D <RgbPixel> image = Dlib.LoadImage <RgbPixel>(imagePath);
using (FrontalFaceDetector fd = Dlib.GetFrontalFaceDetector())
{
var faces = fd.Operator(image);
foreach (DlibDotNet.Rectangle face in faces)
{
FullObjectDetection shape = _ShapePredictor.Detect(image, face);
ChipDetails faceChipDetail = Dlib.GetFaceChipDetails(shape, 150, 0.25);
Array2D <RgbPixel> faceChip = Dlib.ExtractImageChip <RgbPixel>(image, faceChipDetail);
Bitmap bitmap1 = faceChip.ToBitmap <RgbPixel>();
MainWindow.main.Statusa1 = bitmap1;
Dlib.DrawRectangle(image, face, color: new RgbPixel(0, 255, 255), thickness: 4);
}
}
Bitmap frame = image.ToBitmap <RgbPixel>();
MainWindow.main.Statusa = frame;
}
private static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("You call this program like this: ");
Console.WriteLine("./dnn_semantic_segmentation_train_ex /path/to/images");
Console.WriteLine();
Console.WriteLine("You will also need a trained 'semantic_segmentation_voc2012net.dnn' file.");
Console.WriteLine("You can either train it yourself (see example program");
Console.WriteLine("dnn_semantic_segmentation_train_ex), or download a");
Console.WriteLine("copy from here: http://dlib.net/files/semantic_segmentation_voc2012net.dnn");
return;
}
try
{
// Read the file containing the trained network from the working directory.
using (var net = LossMulticlassLogPerPixel.Deserialize("semantic_segmentation_voc2012net.dnn"))
{
// Show inference results in a window.
using (var win = new ImageWindow())
{
// Find supported image files.
var files = Directory.GetFiles(args[0])
.Where(s => s.EndsWith(".jpeg") || s.EndsWith(".jpg") || s.EndsWith(".png")).ToArray();
Console.WriteLine($"Found {files.Length} images, processing...");
foreach (var file in files)
{
// Load the input image.
using (var inputImage = Dlib.LoadImageAsMatrix <RgbPixel>(file))
{
// Create predictions for each pixel. At this point, the type of each prediction
// is an index (a value between 0 and 20). Note that the net may return an image
// that is not exactly the same size as the input.
using (var output = net.Operator(inputImage))
using (var temp = output.First())
{
// Crop the returned image to be exactly the same size as the input.
var rect = Rectangle.CenteredRect((int)(temp.Columns / 2d), (int)(temp.Rows / 2d), (uint)inputImage.Columns, (uint)inputImage.Rows);
using (var dims = new ChipDims((uint)inputImage.Rows, (uint)inputImage.Columns))
using (var chipDetails = new ChipDetails(rect, dims))
using (var indexLabelImage = Dlib.ExtractImageChip <ushort>(temp, chipDetails, InterpolationTypes.NearestNeighbor))
{
// Convert the indexes to RGB values.
using (var rgbLabelImage = IndexLabelImageToRgbLabelImage(indexLabelImage))
{
// Show the input image on the left, and the predicted RGB labels on the right.
using (var joinedRow = Dlib.JoinRows(inputImage, rgbLabelImage))
{
win.SetImage(joinedRow);
// Find the most prominent class label from amongst the per-pixel predictions.
var classLabel = GetMostProminentNonBackgroundClassLabel(indexLabelImage);
Console.WriteLine($"{file} : {classLabel} - hit enter to process the next image");
Console.ReadKey();
}
}
}
}
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
// Calculate the per-pixel accuracy on a dataset whose file names are supplied as a parameter.
private static double CalculateAccuracy(LossMulticlassLogPerPixel anet, IEnumerable <ImageInfo> dataset)
{
var numRight = 0;
var numWrong = 0;
foreach (var imageInfo in dataset)
{
// Load the input image.
using (var inputImage = Dlib.LoadImageAsMatrix <RgbPixel>(imageInfo.ImageFilename))
{
// Load the ground-truth (RGB) labels.;
using (var rgbLabelImage = Dlib.LoadImageAsMatrix <RgbPixel>(imageInfo.ClassLabelFilename))
{
// Create predictions for each pixel. At this point, the type of each prediction
// is an index (a value between 0 and 20). Note that the net may return an image
// that is not exactly the same size as the input.
using (var output = anet.Operator(inputImage))
using (var temp = output.First())
{
// Convert the indexes to RGB values.
using (var indexLabelImage = new Matrix <ushort>())
{
PascalVOC2012.RgbLabelImageToIndexLabelImage(rgbLabelImage, indexLabelImage);
// Crop the net output to be exactly the same size as the input.
using (var chipDims = new ChipDims((uint)inputImage.Rows, (uint)inputImage.Columns))
using (var chipDetails = new ChipDetails(Dlib.CenteredRect(temp.Columns / 2, temp.Rows / 2,
(uint)inputImage.Columns,
(uint)inputImage.Rows),
chipDims))
{
using (var netOutput = Dlib.ExtractImageChip <ushort>(temp, chipDetails, InterpolationTypes.NearestNeighbor))
{
var nr = indexLabelImage.Rows;
var nc = indexLabelImage.Columns;
// Compare the predicted values to the ground-truth values.
for (var r = 0; r < nr; ++r)
{
for (var c = 0; c < nc; ++c)
{
var truth = indexLabelImage[r, c];
if (truth != LossMulticlassLogPerPixel.LabelToIgnore)
{
var prediction = netOutput[r, c];
if (prediction == truth)
{
++numRight;
}
else
{
++numWrong;
}
}
}
}
}
}
}
}
}
}
}
// Return the accuracy estimate.
return(numRight / (double)(numRight + numWrong));
}
private static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("You call this program like this: ");
Console.WriteLine("./dnn_instance_segmentation_train_ex /path/to/images");
Console.WriteLine();
Console.WriteLine($"You will also need a trained '{InstanceSegmentationNetFilename}' file.");
Console.WriteLine("You can either train it yourself (see example program");
Console.WriteLine("dnn_instance_segmentation_train_ex), or download a");
Console.WriteLine($"copy from here: http://dlib.net/files/{InstanceSegmentationNetFilename}");
return;
}
try
{
// Read the file containing the trained network from the working directory.
using (var deserialize = new ProxyDeserialize(InstanceSegmentationNetFilename))
using (var detNet = LossMmod.Deserialize(deserialize, 4))
{
var segNetsByClass = new Dictionary <string, LossMulticlassLogPerPixel>();
segNetsByClass.Deserialize(deserialize, 4);
// Show inference results in a window.
using (var win = new ImageWindow())
{
// Find supported image files.
var files = Directory.GetFiles(args[0])
.Where(s => s.EndsWith(".jpeg") || s.EndsWith(".jpg") || s.EndsWith(".png")).ToArray();
using (var rnd = new Rand())
{
Console.WriteLine($"Found {files.Length} images, processing...");
foreach (var file in files.Select(s => new FileInfo(s)))
{
// Load the input image.
using (var inputImage = Dlib.LoadImageAsMatrix <RgbPixel>(file.FullName))
{
// Create predictions for each pixel. At this point, the type of each prediction
// is an index (a value between 0 and 20). Note that the net may return an image
// that is not exactly the same size as the input.
using (var output = detNet.Operator(inputImage))
{
var instances = output.First().ToList();
instances.Sort((lhs, rhs) => (int)lhs.Rect.Area - (int)rhs.Rect.Area);
using (var rgbLabelImage = new Matrix <RgbPixel>())
{
rgbLabelImage.SetSize(inputImage.Rows, inputImage.Columns);
rgbLabelImage.Assign(Enumerable.Range(0, rgbLabelImage.Size).Select(i => new RgbPixel(0, 0, 0)).ToArray());
var foundSomething = false;
foreach (var instance in instances)
{
if (!foundSomething)
{
Console.Write("Found ");
foundSomething = true;
}
else
{
Console.Write(", ");
}
Console.Write(instance.Label);
var croppingRect = GetCroppingRect(instance.Rect);
using (var dims = new ChipDims(SegDim, SegDim))
using (var chipDetails = new ChipDetails(croppingRect, dims))
using (var inputChip = Dlib.ExtractImageChip <RgbPixel>(inputImage, chipDetails, InterpolationTypes.Bilinear))
{
if (!segNetsByClass.TryGetValue(instance.Label, out var i))
{
// per-class segmentation net not found, so we must be using the same net for all classes
// (see bool separate_seg_net_for_each_class in dnn_instance_segmentation_train_ex.cpp)
if (segNetsByClass.Count == 1)
{
throw new ApplicationException();
}
if (string.IsNullOrEmpty(segNetsByClass.First().Key))
{
throw new ApplicationException();
}
}
var segNet = i != null
? i // use the segmentation net trained for this class
: segNetsByClass.First().Value; // use the same segmentation net for all classes
using (var mask = segNet.Operator(inputChip))
{
var randomColor = new RgbPixel(
rnd.GetRandom8BitNumber(),
rnd.GetRandom8BitNumber(),
rnd.GetRandom8BitNumber()
);
using (var resizedMask = new Matrix <ushort>((int)chipDetails.Rect.Height, (int)chipDetails.Rect.Width))
{
Dlib.ResizeImage(mask.First(), resizedMask);
for (int r = 0, nr = resizedMask.Rows; r < nr; ++r)
{
for (int c = 0, nc = resizedMask.Columns; c < nc; ++c)
{
if (resizedMask[r, c] != 0)
{
var y = (int)(chipDetails.Rect.Top + r);
var x = (int)(chipDetails.Rect.Left + c);
if (y >= 0 && y < rgbLabelImage.Rows && x >= 0 && x < rgbLabelImage.Columns)
{
rgbLabelImage[y, x] = randomColor;
}
}
}
}
}
var voc2012Class = PascalVOC2012.FindVoc2012Class(instance.Label);
Dlib.DrawRectangle(rgbLabelImage, instance.Rect, voc2012Class.RgbLabel, 1u);
}
}
}
instances.DisposeElement();
using (var tmp = Dlib.JoinRows(inputImage, rgbLabelImage))
{
// Show the input image on the left, and the predicted RGB labels on the right.
win.SetImage(tmp);
if (instances.Any())
{
Console.Write($" in {file.Name} - hit enter to process the next image");
Console.ReadKey();
}
}
}
}
}
}
}
}
foreach (var kvp in segNetsByClass)
{
kvp.Value.Dispose();
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
private static int ClusterDataset(CommandLineApplication commandLine)
{
// make sure the user entered an argument to this program
if (commandLine.RemainingArguments.Count != 1)
{
Console.WriteLine("The --cluster option requires you to give one XML file on the command line.");
return(ExitFailure);
}
var clusterOption = commandLine.Option("-cluster|--cluster", "", CommandOptionType.SingleValue);
var sizeOption = commandLine.Option("-size|--size", "", CommandOptionType.SingleValue);
var clusterValue = clusterOption.HasValue() ? clusterOption.Value() : "2";
var sizeValue = sizeOption.HasValue() ? sizeOption.Value() : "8000";
if (!uint.TryParse(clusterValue, out var numClusters))
{
return(ExitFailure);
}
if (!uint.TryParse(sizeValue, out var chipSize))
{
return(ExitFailure);
}
var argument = commandLine.RemainingArguments[0];
using (var data = Dlib.ImageDatasetMetadata.LoadImageDatasetMetadata(argument))
{
Environment.CurrentDirectory = Path.GetDirectoryName(argument);
double aspectRatio = MeanAspectRatio(data);
var images = new Array <Array2D <RgbPixel> >();
var feats = new List <Matrix <double> >();
var dataImages = data.Images;
//console_progress_indicator pbar(dataImages.Length);
// extract all the object chips and HOG features.
Console.WriteLine("Loading image data...");
for (var i = 0; i < dataImages.Length; ++i)
{
//pbar.print_status(i);
if (!HasNonIgnoredBoxes(dataImages[i]))
{
continue;
}
using (var img = Dlib.LoadImage <RgbPixel>(dataImages[i].FileName))
{
var boxes = dataImages[i].Boxes;
for (var j = 0; j < boxes.Length; ++j)
{
if (boxes[j].Ignore || boxes[j].Rect.Area < 10)
{
continue;
}
var rect = new DRectangle(boxes[j].Rect);
rect = DRectangle.SetAspectRatio(rect, aspectRatio);
using (var chipDetail = new ChipDetails(rect, chipSize))
{
var chip = Dlib.ExtractImageChip <RgbPixel>(img, chipDetail);
Dlib.ExtractFHogFeatures <RgbPixel>(chip, out var feature);
feats.Add(feature);
images.PushBack(chip);
}
}
}
}
if (!feats.Any())
{
Console.WriteLine("No non-ignored object boxes found in the XML dataset. You can't cluster an empty dataset.");
return(ExitFailure);
}
Console.WriteLine("\nClustering objects...");
var assignments = AngularCluster(feats, numClusters).ToList();
// Now output each cluster to disk as an XML file.
for (uint c = 0; c < numClusters; ++c)
{
// We are going to accumulate all the image metadata for cluster c. We put it
// into idata so we can sort the images such that images with central chips
// come before less central chips. The idea being to get the good chips to
// show up first in the listing, making it easy to manually remove bad ones if
// that is desired.
var idata = new List <Pair <double, Image> >(dataImages.Length);
var idx = 0;
for (var i = 0; i < dataImages.Length; ++i)
{
idata.Add(new Pair <double, Image> {
Second = new Image()
});
idata[i].First = double.PositiveInfinity;
idata[i].Second.FileName = dataImages[i].FileName;
if (!HasNonIgnoredBoxes(dataImages[i]))
{
continue;
}
var idataBoxes = new List <Box>();
var boxes = dataImages[i].Boxes;
for (var j = 0; j < boxes.Length; ++j)
{
idataBoxes.Add(boxes[j]);
if (boxes[j].Ignore || boxes[j].Rect.Area < 10)
{
continue;
}
// If this box goes into cluster c then update the score for the whole
// image based on this boxes' score. Otherwise, mark the box as
// ignored.
if (assignments[idx].C == c)
{
idata[i].First = Math.Min(idata[i].First, assignments[idx].Distance);
}
else
{
idataBoxes.Last().Ignore = true;
}
++idx;
}
idata[i].Second.Boxes = idataBoxes.ToArray();
}
// now save idata to an xml file.
idata.Sort((a, b) =>
{
var diff = a.First - b.First;
return(diff > 0 ? 1 : diff < 0 ? -1 : 0);
});
using (var cdata = new Dataset())
{
cdata.Comment = $"{data.Comment}\n\n This file contains objects which were clustered into group {c + 1} of {numClusters} groups with a chip size of {chipSize} by imglab.";
cdata.Name = data.Name;
var cdataImages = new List <Image>();
for (var i = 0; i < idata.Count; ++i)
{
// if this image has non-ignored boxes in it then include it in the output.
if (!double.IsPositiveInfinity(idata[i].First))
{
cdataImages.Add(idata[i].Second);
}
}
cdata.Images = cdataImages.ToArray();
var outfile = $"cluster_{c + 1:D3}.xml";
Console.WriteLine($"Saving {outfile}");
Dlib.ImageDatasetMetadata.SaveImageDatasetMetadata(cdata, outfile);
}
}
// Now output each cluster to disk as a big tiled jpeg file. Sort everything so, just
// like in the xml file above, the best objects come first in the tiling.
assignments.Sort();
for (uint c = 0; c < numClusters; ++c)
{
var temp = new Array <Array2D <RgbPixel> >();
for (var i = 0; i < assignments.Count(); ++i)
{
if (assignments[i].C == c)
{
temp.PushBack(images[(int)assignments[i].Index]);
}
}
var outfile = $"cluster_{c + 1:D3}.jpg";
Console.WriteLine($"Saving {outfile}");
using (var tile = Dlib.TileImages(temp))
Dlib.SaveJpeg(tile, outfile);
}
}
return(ExitSuccess);
}
public void ExtractImageChip()
{
const string testName = nameof(ExtractImageChip);
var path = this.GetDataFile($"{LoadTarget}.bmp");
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = false },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
var type = this.GetType().Name;
using (var dims = new ChipDims(227, 227))
using (var chip = new ChipDetails(new Rectangle(0, 0, 100, 100), dims))
foreach (var input in tests)
{
var expectResult = input.ExpectResult;
var imageObj = DlibTest.LoadImage(input.Type, path);
var outputImageAction = new Func <bool, Array2DBase>(expect =>
{
switch (input.Type)
{
case ImageTypes.RgbPixel:
return(Dlib.ExtractImageChip <RgbPixel>(imageObj, chip));
case ImageTypes.RgbAlphaPixel:
return(Dlib.ExtractImageChip <RgbAlphaPixel>(imageObj, chip));
case ImageTypes.UInt8:
return(Dlib.ExtractImageChip <byte>(imageObj, chip));
case ImageTypes.UInt16:
return(Dlib.ExtractImageChip <ushort>(imageObj, chip));
case ImageTypes.UInt32:
return(Dlib.ExtractImageChip <uint>(imageObj, chip));
case ImageTypes.Int8:
return(Dlib.ExtractImageChip <sbyte>(imageObj, chip));
case ImageTypes.Int16:
return(Dlib.ExtractImageChip <short>(imageObj, chip));
case ImageTypes.Int32:
return(Dlib.ExtractImageChip <int>(imageObj, chip));
case ImageTypes.HsiPixel:
return(Dlib.ExtractImageChip <HsiPixel>(imageObj, chip));
case ImageTypes.Float:
return(Dlib.ExtractImageChip <float>(imageObj, chip));
case ImageTypes.Double:
return(Dlib.ExtractImageChip <double>(imageObj, chip));
default:
throw new ArgumentOutOfRangeException();
}
});
var successAction = new Action <Array2DBase>(image =>
{
Dlib.SaveBmp(image, $"{Path.Combine(this.GetOutDir(type, testName), $"{LoadTarget}_{input.Type}.bmp")}");
});
var failAction = new Action(() =>
{
Assert.Fail($"{testName} should throw exception for InputType: {input.Type}.");
});
var finallyAction = new Action(() =>
{
if (imageObj != null)
{
this.DisposeAndCheckDisposedState(imageObj);
}
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {input.Type}, Type: {input.Type}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
