public static YCbCrPixel Convert(RgbPixel rgb)
{
var yChannel = 0.299 * rgb.R + 0.587 * rgb.G + 0.114 * rgb.B;
var cbChannel = -0.168935 * rgb.R - 0.331665 * rgb.G + 0.50059 * rgb.B;
var crChannel = 0.499813 * rgb.R - 0.418531 * rgb.G - 0.081282 * rgb.B;
var yCbCr = new YCbCrPixel(yChannel, cbChannel, crChannel);
return(yCbCr);
}
private static bool IsInstancePixel(RgbPixel rgbLabel)
{
if (rgbLabel == new RgbPixel(0, 0, 0))
{
return(false); // Background
}
if (rgbLabel == new RgbPixel(224, 224, 192))
{
return(false); // The cream-colored `void' label is used in border regions and to mask difficult objects
}
return(true);
}
public static RgbPixel[,] Convert(YCbCrPixel[,] matrix)
{
var height = matrix.GetLength(0);
var width = matrix.GetLength(1);
var convertedMatrix = new RgbPixel[height, width];
for (var i = 0; i < height; i++)
{
for (var j = 0; j < width; j++)
{
convertedMatrix[i, j] = YCbCrToRgbConverter.Convert(matrix[i, j]);
}
}
return(convertedMatrix);
}
public void Test()
{
var bmp1 = (Bitmap)Image.FromFile(@"C:\Users\FessEmpty\Downloads\parallelprogramming-shpora2016-56f562c98bbf\parallelprogramming-shpora2016-56f562c98bbf\JPEG\sample.bmp");
var bitmapData1 = bmp1.LockBits(new Rectangle(0, 0, bmp1.Width, bmp1.Height), ImageLockMode.ReadOnly, bmp1.PixelFormat);
const int rgpPixelSize = 3;
var additional = bitmapData1.Stride - bitmapData1.Width * rgpPixelSize;
var result = new RgbPixel[bmp1.Height, bmp1.Width];
unsafe
{
var imagePointer = (byte *)bitmapData1.Scan0;
for (var j = 0; j < bitmapData1.Height; j++, imagePointer += additional)
{
for (var i = 0; i < bitmapData1.Width; i++, imagePointer += rgpPixelSize)
{
result[j, i] = new RgbPixel(imagePointer[0], imagePointer[1], imagePointer[2]);
}
}
}
bmp1.UnlockBits(bitmapData1);
var convert = MatrixRgbToYCbCrConveter.Convert(result);
var back = MatrixYCbCrToRgbConverter.Convert(convert);
var bmp2 = new Bitmap(bmp1.Width, bmp1.Height, bmp1.PixelFormat);
var bitmapData2 = bmp2.LockBits(new Rectangle(0, 0, bmp2.Width, bmp2.Height), ImageLockMode.ReadOnly, bmp2.PixelFormat);
unsafe
{
var imagePointer = (byte *)bitmapData2.Scan0;
for (var j = 0; j < bitmapData2.Height; j++, imagePointer += additional)
{
for (var i = 0; i < bitmapData2.Width; i++, imagePointer += rgpPixelSize)
{
imagePointer[0] = back[j, i].R;
imagePointer[1] = back[j, i].G;
imagePointer[2] = back[j, i].B;
}
}
}
bmp2.UnlockBits(bitmapData2);
bmp2.Save(@"C:\Users\FessEmpty\Downloads\parallelprogramming-shpora2016-56f562c98bbf\parallelprogramming-shpora2016-56f562c98bbf\JPEG\sampleTest.bmp", ImageFormat.Bmp);
}
private Array2D <RgbPixel> ToArray(Mat mat)
{
var array = new Array2D <RgbPixel>(mat.Rows, mat.Cols);
using (var mat3 = new MatOfByte3(mat))
{
var indexer = mat3.GetIndexer();
for (var i = 0; i < array.Rows; i++)
{
var destRow = array[i];
for (var j = 0; j < array.Columns; j++)
{
var color = indexer[i, j];
destRow[j] = new RgbPixel(color.Item2, color.Item1, color.Item0);
}
}
}
return(array);
}
private static void DrawDebugDataOnImage(
Array2D <RgbPixel> img,
FullObjectDetection shape,
Rectangle face_rect,
Rectangle left_eye_rect,
Rectangle right_eye_rect)
{
if (debugData)
{
// For debugging, show face points
for (uint i = 0; i < shape.Parts; ++i)
{
var point = shape.GetPart(i);
var rect = new Rectangle(point);
var color = new RgbPixel(255, 255, 0);
if (i >= 36 && i <= 41)
{
color = new RgbPixel(0, 0, 255);
}
else if (i >= 42 && i <= 47)
{
color = new RgbPixel(0, 255, 0);
}
if (i >= 27 && i <= 35)
{
color = new RgbPixel(255, 0, 0);
}
Dlib.DrawRectangle(img, rect, color: color, thickness: 4);
}
Dlib.DrawRectangle(img, face_rect, color: new RgbPixel(0, 255, 0), thickness: 4);
Dlib.DrawRectangle(img, left_eye_rect, color: new RgbPixel(0, 255, 255), thickness: 4);
Dlib.DrawRectangle(img, right_eye_rect, color: new RgbPixel(0, 255, 255), thickness: 4);
}
}
private static Matrix <ushort> KeepOnlyCurrentInstance(Matrix <RgbPixel> rgbLabelImage, RgbPixel rgbLabel)
{
var nr = rgbLabelImage.Rows;
var nc = rgbLabelImage.Columns;
var result = new Matrix <ushort>(nr, nc);
for (var r = 0; r < nr; ++r)
{
for (var c = 0; c < nc; ++c)
{
var index = rgbLabelImage[r, c];
if (index == rgbLabel)
{
result[r, c] = 1;
}
else if (index == new RgbPixel(224, 224, 192))
{
result[r, c] = LossMulticlassLogPerPixel.LabelToIgnore;
}
else
{
result[r, c] = 0;
}
}
}
return(result);
}
private static void Load(string type, string directory, string meanImage, out IList <Matrix <RgbPixel> > images, out IList <uint> labels)
{
Matrix <RgbPixel> mean = null;
try
{
if (File.Exists(meanImage))
{
mean = Dlib.LoadImageAsMatrix <RgbPixel>(meanImage);
}
var csv = ReadCsv(Path.Combine(directory, $"{type}.csv"));
var imageList = new List <Matrix <RgbPixel> >();
var labelList = new List <uint>();
foreach (var kvp in csv)
{
var path = Path.Combine(directory, type, kvp.Key);
if (!File.Exists(path))
{
continue;
}
using (var tmp = Dlib.LoadImageAsMatrix <RgbPixel>(path))
{
if (mean != null)
{
using (var m = new Matrix <RgbPixel>(Size, Size))
{
Dlib.ResizeImage(tmp, m);
// ToDo: Support subtract operator on DlibDotNet
// var ret = m - mean;
var ret = new Matrix <RgbPixel>(Size, Size);
for (var row = 0; row < Size; row++)
{
for (var col = 0; col < Size; col++)
{
var left = m[row, col];
var right = mean[row, col];
var red = left.Red - right.Red;
var green = left.Green - right.Green;
var blue = left.Blue - right.Blue;
ret[row, col] = new RgbPixel((byte)red, (byte)green, (byte)blue);
}
}
imageList.Add(ret);
}
}
else
{
var m = new Matrix <RgbPixel>(Size, Size);
Dlib.ResizeImage(tmp, m);
imageList.Add(m);
}
labelList.Add(kvp.Value);
}
}
images = imageList;
labels = labelList;
}
finally
{
mean?.Dispose();
}
}
private static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("You call this program like this: ");
Console.WriteLine("./dnn_instance_segmentation_train_ex /path/to/images");
Console.WriteLine();
Console.WriteLine($"You will also need a trained '{InstanceSegmentationNetFilename}' file.");
Console.WriteLine("You can either train it yourself (see example program");
Console.WriteLine("dnn_instance_segmentation_train_ex), or download a");
Console.WriteLine($"copy from here: http://dlib.net/files/{InstanceSegmentationNetFilename}");
return;
}
try
{
// Read the file containing the trained network from the working directory.
using (var deserialize = new ProxyDeserialize(InstanceSegmentationNetFilename))
using (var detNet = LossMmod.Deserialize(deserialize, 4))
{
var segNetsByClass = new Dictionary <string, LossMulticlassLogPerPixel>();
segNetsByClass.Deserialize(deserialize, 4);
// Show inference results in a window.
using (var win = new ImageWindow())
{
// Find supported image files.
var files = Directory.GetFiles(args[0])
.Where(s => s.EndsWith(".jpeg") || s.EndsWith(".jpg") || s.EndsWith(".png")).ToArray();
using (var rnd = new Rand())
{
Console.WriteLine($"Found {files.Length} images, processing...");
foreach (var file in files.Select(s => new FileInfo(s)))
{
// Load the input image.
using (var inputImage = Dlib.LoadImageAsMatrix <RgbPixel>(file.FullName))
{
// Create predictions for each pixel. At this point, the type of each prediction
// is an index (a value between 0 and 20). Note that the net may return an image
// that is not exactly the same size as the input.
using (var output = detNet.Operator(inputImage))
{
var instances = output.First().ToList();
instances.Sort((lhs, rhs) => (int)lhs.Rect.Area - (int)rhs.Rect.Area);
using (var rgbLabelImage = new Matrix <RgbPixel>())
{
rgbLabelImage.SetSize(inputImage.Rows, inputImage.Columns);
rgbLabelImage.Assign(Enumerable.Range(0, rgbLabelImage.Size).Select(i => new RgbPixel(0, 0, 0)).ToArray());
var foundSomething = false;
foreach (var instance in instances)
{
if (!foundSomething)
{
Console.Write("Found ");
foundSomething = true;
}
else
{
Console.Write(", ");
}
Console.Write(instance.Label);
var croppingRect = GetCroppingRect(instance.Rect);
using (var dims = new ChipDims(SegDim, SegDim))
using (var chipDetails = new ChipDetails(croppingRect, dims))
using (var inputChip = Dlib.ExtractImageChip <RgbPixel>(inputImage, chipDetails, InterpolationTypes.Bilinear))
{
if (!segNetsByClass.TryGetValue(instance.Label, out var i))
{
// per-class segmentation net not found, so we must be using the same net for all classes
// (see bool separate_seg_net_for_each_class in dnn_instance_segmentation_train_ex.cpp)
if (segNetsByClass.Count == 1)
{
throw new ApplicationException();
}
if (string.IsNullOrEmpty(segNetsByClass.First().Key))
{
throw new ApplicationException();
}
}
var segNet = i != null
? i // use the segmentation net trained for this class
: segNetsByClass.First().Value; // use the same segmentation net for all classes
using (var mask = segNet.Operator(inputChip))
{
var randomColor = new RgbPixel(
rnd.GetRandom8BitNumber(),
rnd.GetRandom8BitNumber(),
rnd.GetRandom8BitNumber()
);
using (var resizedMask = new Matrix <ushort>((int)chipDetails.Rect.Height, (int)chipDetails.Rect.Width))
{
Dlib.ResizeImage(mask.First(), resizedMask);
for (int r = 0, nr = resizedMask.Rows; r < nr; ++r)
{
for (int c = 0, nc = resizedMask.Columns; c < nc; ++c)
{
if (resizedMask[r, c] != 0)
{
var y = (int)(chipDetails.Rect.Top + r);
var x = (int)(chipDetails.Rect.Left + c);
if (y >= 0 && y < rgbLabelImage.Rows && x >= 0 && x < rgbLabelImage.Columns)
{
rgbLabelImage[y, x] = randomColor;
}
}
}
}
}
var voc2012Class = PascalVOC2012.FindVoc2012Class(instance.Label);
Dlib.DrawRectangle(rgbLabelImage, instance.Rect, voc2012Class.RgbLabel, 1u);
}
}
}
instances.DisposeElement();
using (var tmp = Dlib.JoinRows(inputImage, rgbLabelImage))
{
// Show the input image on the left, and the predicted RGB labels on the right.
win.SetImage(tmp);
if (instances.Any())
{
Console.Write($" in {file.Name} - hit enter to process the next image");
Console.ReadKey();
}
}
}
}
}
}
}
}
foreach (var kvp in segNetsByClass)
{
kvp.Value.Dispose();
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public FaceRecognizer(string imageName, RgbPixel penColor, ModelPathSystem pathSystem) // Get image from image file
{
PathSystem = pathSystem;
Image = Dlib.LoadImage <RgbPixel>(Path.Combine(PathSystem.InputsPath, imageName));
Color = penColor;
}
// The PASCAL VOC2012 dataset contains 20 ground-truth classes + background. Each class
// is represented using an RGB color value. We associate each class also to an index in the
// range [0, 20], used internally by the network. To convert the ground-truth data to
// something that the network can efficiently digest, we need to be able to map the RGB
// values to the corresponding indexes.
// Given an RGB representation, find the corresponding PASCAL VOC2012 class
// (e.g., 'dog').
private static Voc2012Class FindVoc2012Class(RgbPixel rgbLabel)
{
return(Common.FindVoc2012Class(@class => rgbLabel == @class.RgbLabel));
}
// Convert an RGB class label to an index in the range [0, 20].
private static ushort RgbLabelToIndexLabel(RgbPixel rgbLabel)
{
return(FindVoc2012Class(rgbLabel).Index);
}
public void LoadImageData2()
{
const int cols = 512;
const int rows = 512;
const int steps = 512;
var tests = new[]
{
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
var random = new Random(0);
foreach (var test in tests)
{
TwoDimentionObjectBase image;
using (var win = new ImageWindow())
{
switch (test.Type)
{
case ImageTypes.UInt8:
{
var data = new byte[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (byte)random.Next(0, 255);
}
}
image = Dlib.LoadImageData <byte>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <byte>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.UInt16:
{
var data = new ushort[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (ushort)random.Next(0, 255);
}
}
image = Dlib.LoadImageData <ushort>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <ushort>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Int16:
{
var data = new short[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (short)random.Next(0, 255);
}
}
image = Dlib.LoadImageData <short>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <short>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Int32:
{
var data = new int[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = random.Next(0, 255);
}
}
image = Dlib.LoadImageData <int>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <int>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Float:
{
var data = new float[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (float)random.NextDouble();
}
}
image = Dlib.LoadImageData <float>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <float>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Double:
{
var data = new double[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (double)random.NextDouble();
}
}
image = Dlib.LoadImageData <double>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <double>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.HsiPixel:
{
var data = new HsiPixel[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = new HsiPixel
{
H = (byte)random.Next(0, 255),
S = (byte)random.Next(0, 255),
I = (byte)random.Next(0, 255)
}
}
}
;
image = Dlib.LoadImageData <HsiPixel>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <HsiPixel>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.RgbPixel:
{
var data = new RgbPixel[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = new RgbPixel
{
Red = (byte)random.Next(0, 255),
Green = (byte)random.Next(0, 255),
Blue = (byte)random.Next(0, 255)
}
}
}
;
image = Dlib.LoadImageData <RgbPixel>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <RgbPixel>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.RgbAlphaPixel:
{
var data = new RgbAlphaPixel[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = new RgbAlphaPixel
{
Red = (byte)random.Next(0, 255),
Green = (byte)random.Next(0, 255),
Blue = (byte)random.Next(0, 255),
Alpha = (byte)random.Next(0, 255)
}
}
}
;
image = Dlib.LoadImageData <RgbAlphaPixel>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <RgbAlphaPixel>)image);
win.WaitUntilClosed();
}
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
Assert.AreEqual(image.Columns, cols, $"Failed to load {test.Type}.");
Assert.AreEqual(image.Rows, rows, $"Failed to load {test.Type}.");
this.DisposeAndCheckDisposedState(image);
}
}
public TrasformazioneAffineRgb(RgbImage <byte> inputImage, double translationX, double translationY, double rotationDegrees, double scaleFactorX, double scaleFactorY, RgbPixel <byte> background, int resultWidth, int resultHeight)
: base(inputImage, translationX, translationY, rotationDegrees, scaleFactorX, scaleFactorY, resultWidth, resultHeight)
{
Background = background;
}
public void Rgb_ConvertToYCbCrAndBack_SameResult()
{
var rgb = new RgbPixel(50, 150, 255);
var yCbCr = RgbToYCbCrConverter.Convert(rgb);
var newRgb = YCbCrToRgbConverter.Convert(yCbCr);
}
} // Model path system
public FaceRecognizer(byte[] image, RgbPixel penColor, ModelPathSystem pathSystem) // Get image from bytes array
{
PathSystem = pathSystem;
Image = image.ToArray2D(PathSystem);
Color = penColor;
}
private static void Main()
{
try
{
// You can get this file from http://dlib.net/files/mmod_rear_end_vehicle_detector.dat.bz2
// This network was produced by the dnn_mmod_train_find_cars_ex.cpp example program.
// As you can see, the file also includes a separately trained shape_predictor. To see
// a generic example of how to train those refer to train_shape_predictor_ex.cpp.
using (var deserialize = new ProxyDeserialize("mmod_rear_end_vehicle_detector.dat"))
using (var net = LossMmod.Deserialize(deserialize, 1))
using (var sp = ShapePredictor.Deserialize(deserialize))
using (var img = Dlib.LoadImageAsMatrix <RgbPixel>("mmod_cars_test_image.jpg"))
using (var win = new ImageWindow())
{
win.SetImage(img);
// Run the detector on the image and show us the output.
var dets = net.Operator(img).First();
foreach (var d in dets)
{
// We use a shape_predictor to refine the exact shape and location of the detection
// box. This shape_predictor is trained to simply output the 4 corner points of
// the box. So all we do is make a rectangle that tightly contains those 4 points
// and that rectangle is our refined detection position.
var fd = sp.Detect(img, d);
var rect = Rectangle.Empty;
for (var j = 0u; j < fd.Parts; ++j)
{
rect += fd.GetPart(j);
}
win.AddOverlay(rect, new RgbPixel(255, 0, 0));
}
Console.WriteLine("Hit enter to view the intermediate processing steps");
Console.ReadKey();
// Now let's look at how the detector works. The high level processing steps look like:
// 1. Create an image pyramid and pack the pyramid into one big image. We call this
// image the "tiled pyramid".
// 2. Run the tiled pyramid image through the CNN. The CNN outputs a new image where
// bright pixels in the output image indicate the presence of cars.
// 3. Find pixels in the CNN's output image with a value > 0. Those locations are your
// preliminary car detections.
// 4. Perform non-maximum suppression on the preliminary detections to produce the
// final output.
//
// We will be plotting the images from steps 1 and 2 so you can visualize what's
// happening. For the CNN's output image, we will use the jet colormap so that "bright"
// outputs, i.e. pixels with big values, appear in red and "dim" outputs appear as a
// cold blue color. To do this we pick a range of CNN output values for the color
// mapping. The specific values don't matter. They are just selected to give a nice
// looking output image.
const float lower = -2.5f;
const float upper = 0.0f;
Console.WriteLine($"jet color mapping range: lower={lower} upper={upper}");
// Create a tiled pyramid image and display it on the screen.
// Get the type of pyramid the CNN used
//using pyramid_type = std::remove_reference < decltype(input_layer(net)) >::type::pyramid_type;
// And tell create_tiled_pyramid to create the pyramid using that pyramid type.
using (var inputLayer = new InputRgbImagePyramid <PyramidDown>(6))
{
net.TryGetInputLayer(inputLayer);
var padding = inputLayer.GetPyramidPadding();
var outerPadding = inputLayer.GetPyramidOuterPadding();
Dlib.CreateTiledPyramid <RgbPixel, PyramidDown>(img,
padding,
outerPadding,
6,
out var tiledImg,
out var rects);
using (var winpyr = new ImageWindow(tiledImg, "Tiled pyramid"))
{
// This CNN detector represents a sliding window detector with 3 sliding windows. Each
// of the 3 windows has a different aspect ratio, allowing it to find vehicles which
// are either tall and skinny, squarish, or short and wide. The aspect ratio of a
// detection is determined by which channel in the output image triggers the detection.
// Here we are just going to max pool the channels together to get one final image for
// our display. In this image, a pixel will be bright if any of the sliding window
// detectors thinks there is a car at that location.
using (var subnet = net.GetSubnet())
{
var output = subnet.Output;
Console.WriteLine($"Number of channels in final tensor image: {output.K}");
var networkOutput = Dlib.ImagePlane(output);
for (var k = 1; k < output.K; k++)
{
using (var tmpNetworkOutput = Dlib.ImagePlane(output, 0, k))
{
var maxPointWise = Dlib.MaxPointWise(networkOutput, tmpNetworkOutput);
networkOutput.Dispose();
networkOutput = maxPointWise;
}
}
// We will also upsample the CNN's output image. The CNN we defined has an 8x
// downsampling layer at the beginning. In the code below we are going to overlay this
// CNN output image on top of the raw input image. To make that look nice it helps to
// upsample the CNN output image back to the same resolution as the input image, which
// we do here.
var networkOutputScale = img.Columns / (double)networkOutput.Columns;
Dlib.ResizeImage(networkOutput, networkOutputScale);
// Display the network's output as a color image.
using (var jet = Dlib.Jet(networkOutput, upper, lower))
using (var winOutput = new ImageWindow(jet, "Output tensor from the network"))
{
// Also, overlay network_output on top of the tiled image pyramid and display it.
for (var r = 0; r < tiledImg.Rows; ++r)
{
for (var c = 0; c < tiledImg.Columns; ++c)
{
var tmp = new DPoint(c, r);
tmp = Dlib.InputTensorToOutputTensor(net, tmp);
var dp = networkOutputScale * tmp;
tmp = new DPoint((int)dp.X, (int)dp.Y);
if (Dlib.GetRect(networkOutput).Contains((int)tmp.X, (int)tmp.Y))
{
var val = networkOutput[(int)tmp.Y, (int)tmp.X];
// alpha blend the network output pixel with the RGB image to make our
// overlay.
var p = new RgbAlphaPixel();
Dlib.AssignPixel(ref p, Dlib.ColormapJet(val, lower, upper));
p.Alpha = 120;
var rgb = new RgbPixel();
Dlib.AssignPixel(ref rgb, p);
tiledImg[r, c] = rgb;
}
}
}
// If you look at this image you can see that the vehicles have bright red blobs on
// them. That's the CNN saying "there is a car here!". You will also notice there is
// a certain scale at which it finds cars. They have to be not too big or too small,
// which is why we have an image pyramid. The pyramid allows us to find cars of all
// scales.
using (var winPyrOverlay = new ImageWindow(tiledImg, "Detection scores on image pyramid"))
{
// Finally, we can collapse the pyramid back into the original image. The CNN doesn't
// actually do this step, since it's enough to threshold the tiled pyramid image to get
// the detections. However, it makes a nice visualization and clearly indicates that
// the detector is firing for all the cars.
using (var collapsed = new Matrix <float>(img.Rows, img.Columns))
using (var inputTensor = new ResizableTensor())
{
inputLayer.ToTensor(img, 1, inputTensor);
for (var r = 0; r < collapsed.Rows; ++r)
{
for (var c = 0; c < collapsed.Columns; ++c)
{
// Loop over a bunch of scale values and look up what part of network_output
// corresponds to the point(c,r) in the original image, then take the max
// detection score over all the scales and save it at pixel point(c,r).
var maxScore = -1e30f;
for (double scale = 1; scale > 0.2; scale *= 5.0 / 6.0)
{
// Map from input image coordinates to tiled pyramid coordinates.
var tensorSpace = inputLayer.ImageSpaceToTensorSpace(inputTensor, scale, new DRectangle(new DPoint(c, r)));
var tmp = tensorSpace.Center;
// Now map from pyramid coordinates to network_output coordinates.
var dp = networkOutputScale * Dlib.InputTensorToOutputTensor(net, tmp);
tmp = new DPoint((int)dp.X, (int)dp.Y);
if (Dlib.GetRect(networkOutput).Contains((int)tmp.X, (int)tmp.Y))
{
var val = networkOutput[(int)tmp.Y, (int)tmp.X];
if (val > maxScore)
{
maxScore = val;
}
}
}
collapsed[r, c] = maxScore;
// Also blend the scores into the original input image so we can view it as
// an overlay on the cars.
var p = new RgbAlphaPixel();
Dlib.AssignPixel(ref p, Dlib.ColormapJet(maxScore, lower, upper));
p.Alpha = 120;
var rgb = new RgbPixel();
Dlib.AssignPixel(ref rgb, p);
img[r, c] = rgb;
}
}
using (var jet2 = Dlib.Jet(collapsed, upper, lower))
using (var winCollapsed = new ImageWindow(jet2, "Collapsed output tensor from the network"))
using (var winImgAndSal = new ImageWindow(img, "Collapsed detection scores on raw image"))
{
Console.WriteLine("Hit enter to end program");
Console.ReadKey();
}
}
}
}
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public TrasformazioneAffineRgb(RgbImage <byte> inputImage, double scaleFactor, RgbPixel <byte> background)
: base(inputImage, scaleFactor)
{
Background = background;
}
public TrasformazioneAffineRgb(RgbImage <byte> inputImage, double translationX, double translationY, double rotationDegrees, RgbPixel <byte> background)
: base(inputImage, translationX, translationY, rotationDegrees)
{
Background = background;
}
private static void Preprocess(string type, string input, FrontalFaceDetector faceDetector, ShapePredictor posePredictor, string output)
{
var imageCount = 0;
var r = new ulong[Size * Size];
var g = new ulong[Size * Size];
var b = new ulong[Size * Size];
var csv = ReadCsv(Path.Combine(input, $"{type}.csv"));
var outputDir = Path.Combine(output, type);
foreach (var kvp in csv)
{
using (var tmp = Dlib.LoadImageAsMatrix <RgbPixel>(Path.Combine(input, type, kvp.Key)))
{
var dets = faceDetector.Operator(tmp);
if (!dets.Any())
{
Console.WriteLine($"Warning: Failed to detect face from '{kvp}'");
continue;
}
// Get max size rectangle. It could be better face.
var det = dets.Select((val, idx) => new { V = val, I = idx }).Aggregate((max, working) => (max.V.Area > working.V.Area) ? max : working).V;
using (var ret = posePredictor.Detect(tmp, det))
using (var chip = Dlib.GetFaceChipDetails(ret, Size, 0d))
using (var faceChips = Dlib.ExtractImageChip <RgbPixel>(tmp, chip))
{
var dst = Path.Combine(outputDir, kvp.Key);
var dstDir = Path.GetDirectoryName(dst);
Directory.CreateDirectory(dstDir);
Dlib.SaveJpeg(faceChips, Path.Combine(outputDir, kvp.Key), 100);
var index = 0;
for (var row = 0; row < Size; row++)
{
for (var col = 0; col < Size; col++)
{
var rgb = faceChips[row, col];
r[index] += rgb.Red;
g[index] += rgb.Green;
b[index] += rgb.Blue;
index++;
}
}
}
imageCount++;
}
}
using (var mean = new Matrix <RgbPixel>(Size, Size))
{
var index = 0;
for (var row = 0; row < Size; row++)
{
for (var col = 0; col < Size; col++)
{
var red = (double)r[index] / imageCount;
var green = (double)g[index] / imageCount;
var blue = (double)b[index] / imageCount;
var newRed = (byte)Math.Round(red);
var newGreen = (byte)Math.Round(green);
var newBlue = (byte)Math.Round(blue);
mean[row, col] = new RgbPixel(newRed, newGreen, newBlue);
index++;
}
}
Dlib.SaveBmp(mean, Path.Combine(output, $"{type}.mean.bmp"));
}
}
public void GetRowColumn()
{
const int width = 150;
const int height = 100;
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
foreach (var test in tests)
{
var array2D = CreateArray2D(test.Type, height, width);
switch (array2D.ImageType)
{
case ImageTypes.UInt8:
{
var array = (Array2D <byte>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<byte> failed");
}
break;
case ImageTypes.UInt16:
{
var array = (Array2D <ushort>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<ushort> failed");
}
break;
case ImageTypes.UInt32:
{
var array = (Array2D <uint>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255u, "Array<uint> failed");
}
break;
case ImageTypes.Int8:
{
var array = (Array2D <sbyte>)array2D;
Dlib.AssignAllPpixels(array, 127);
using (var row = array[0])
Assert.AreEqual(row[0], 127, "Array<sbyte> failed");
}
break;
case ImageTypes.Int16:
{
var array = (Array2D <short>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<short> failed");
}
break;
case ImageTypes.Int32:
{
var array = (Array2D <int>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<int> failed");
}
break;
case ImageTypes.Float:
{
var array = (Array2D <float>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<float> failed");
}
break;
case ImageTypes.Double:
{
var array = (Array2D <double>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<double> failed");
}
break;
case ImageTypes.RgbPixel:
{
var array = (Array2D <RgbPixel>)array2D;
var pixel = new RgbPixel
{
Red = 255,
Blue = 255,
Green = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.Red, 255, "Array<RgbPixel> failed");
Assert.AreEqual(t.Blue, 255, "Array<RgbPixel> failed");
Assert.AreEqual(t.Green, 255, "Array<RgbPixel> failed");
}
}
break;
case ImageTypes.RgbAlphaPixel:
{
var array = (Array2D <RgbAlphaPixel>)array2D;
var pixel = new RgbAlphaPixel
{
Red = 255,
Blue = 255,
Green = 255,
Alpha = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.Red, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Blue, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Green, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Alpha, 255, "Array<RgbAlphaPixel> failed");
}
}
break;
case ImageTypes.HsiPixel:
{
var array = (Array2D <HsiPixel>)array2D;
var pixel = new HsiPixel
{
H = 255,
S = 255,
I = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.H, 255, "Array<HsiPixel> failed");
Assert.AreEqual(t.S, 255, "Array<HsiPixel> failed");
Assert.AreEqual(t.I, 255, "Array<HsiPixel> failed");
}
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(array2D.ImageType), array2D.ImageType, null);
}
this.DisposeAndCheckDisposedState(array2D);
}
}
private static void Main(string[] args)
{
try
{
if (args.Length != 2)
{
Console.WriteLine("Call this program like this:");
Console.WriteLine("./dnn_mmod_dog_hipsterizer mmod_dog_hipsterizer.dat faces/dogs.jpg");
Console.WriteLine("You can get the mmod_dog_hipsterizer.dat file from:");
Console.WriteLine("http://dlib.net/files/mmod_dog_hipsterizer.dat.bz2");
return;
}
// load the models as well as glasses and mustache.
using (var deserialize = new ProxyDeserialize(args[0]))
using (var net = LossMmod.Deserialize(deserialize))
using (var sp = ShapePredictor.Deserialize(deserialize))
using (var glasses = Matrix <RgbAlphaPixel> .Deserialize(deserialize))
using (var mustache = Matrix <RgbAlphaPixel> .Deserialize(deserialize))
{
Dlib.PyramidUp(glasses);
Dlib.PyramidUp(mustache);
using (var win1 = new ImageWindow(glasses))
using (var win2 = new ImageWindow(mustache))
using (var winWireframe = new ImageWindow())
using (var winHipster = new ImageWindow())
{
// Now process each image, find dogs, and hipsterize them by drawing glasses and a
// mustache on each dog :)
for (var i = 1; i < args.Length; ++i)
{
using (var img = Dlib.LoadImageAsMatrix <RgbPixel>(args[i]))
{
// Upsampling the image will allow us to find smaller dog faces but will use more
// computational resources.
//pyramid_up(img);
var dets = net.Operator(img).First();
winWireframe.ClearOverlay();
winWireframe.SetImage(img);
// We will also draw a wireframe on each dog's face so you can see where the
// shape_predictor is identifying face landmarks.
var lines = new List <ImageWindow.OverlayLine>();
foreach (var d in dets)
{
// get the landmarks for this dog's face
var shape = sp.Detect(img, d.Rect);
var color = new RgbPixel(0, 255, 0);
var top = shape.GetPart(0);
var leftEar = shape.GetPart(1);
var leftEye = shape.GetPart(2);
var nose = shape.GetPart(3);
var rightEar = shape.GetPart(4);
var rightEye = shape.GetPart(5);
// The locations of the left and right ends of the mustache.
var leftMustache = 1.3 * (leftEye - rightEye) / 2 + nose;
var rightMustache = 1.3 * (rightEye - leftEye) / 2 + nose;
// Draw the glasses onto the image.
var from = new[]
{
2 * new Point(176, 36), 2 * new Point(59, 35)
};
var to = new[]
{
leftEye, rightEye
};
using (var transform = Dlib.FindSimilarityTransform(from, to))
for (uint r = 0, nr = (uint)glasses.Rows; r < nr; ++r)
{
for (uint c = 0, nc = (uint)glasses.Columns; c < nc; ++c)
{
var p = (Point)transform.Operator(new DPoint(c, r));
if (Dlib.GetRect(img).Contains(p))
{
var rgb = img[p.Y, p.X];
Dlib.AssignPixel(ref rgb, glasses[(int)r, (int)c]);
img[p.Y, p.X] = rgb;
}
}
}
// Draw the mustache onto the image right under the dog's nose.
var mustacheRect = Dlib.GetRect(mustache);
from = new[]
{
mustacheRect.TopLeft, mustacheRect.TopRight
};
to = new[]
{
rightMustache, leftMustache
};
using (var transform = Dlib.FindSimilarityTransform(from, to))
for (uint r = 0, nr = (uint)mustache.Rows; r < nr; ++r)
{
for (uint c = 0, nc = (uint)mustache.Columns; c < nc; ++c)
{
var p = (Point)transform.Operator(new DPoint(c, r));
if (Dlib.GetRect(img).Contains(p))
{
var rgb = img[p.Y, p.X];
Dlib.AssignPixel(ref rgb, mustache[(int)r, (int)c]);
img[p.Y, p.X] = rgb;
}
}
}
// Record the lines needed for the face wire frame.
lines.Add(new ImageWindow.OverlayLine(leftEye, nose, color));
lines.Add(new ImageWindow.OverlayLine(nose, rightEye, color));
lines.Add(new ImageWindow.OverlayLine(rightEye, leftEye, color));
lines.Add(new ImageWindow.OverlayLine(rightEye, rightEar, color));
lines.Add(new ImageWindow.OverlayLine(rightEar, top, color));
lines.Add(new ImageWindow.OverlayLine(top, leftEar, color));
lines.Add(new ImageWindow.OverlayLine(leftEar, leftEye, color));
winWireframe.AddOverlay(lines);
winHipster.SetImage(img);
}
Console.WriteLine("Hit enter to process the next image.");
Console.ReadKey();
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public void Indexer3()
{
try
{
using (var matrix = new Matrix <byte>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (byte)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(byte)}");
}
try
{
using (var matrix = new Matrix <ushort>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (ushort)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(ushort)}");
}
try
{
using (var matrix = new Matrix <uint>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (uint)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(uint)}");
}
try
{
using (var matrix = new Matrix <sbyte>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (sbyte)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(sbyte)}");
}
try
{
using (var matrix = new Matrix <short>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (short)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(short)}");
}
try
{
using (var matrix = new Matrix <int>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (int)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(int)}");
}
try
{
using (var matrix = new Matrix <float>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (float)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(float)}");
}
try
{
using (var matrix = new Matrix <double>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (double)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(double)}");
}
try
{
using (var matrix = new Matrix <RgbPixel>(1, 3))
for (var index = 0; index < 3; index++)
{
var b = (byte)(index);
var v = new RgbPixel {
Red = b, Blue = b, Green = b
};
matrix[index] = v;
Assert.AreEqual(v.Red, matrix[index].Red);
Assert.AreEqual(v.Blue, matrix[index].Blue);
Assert.AreEqual(v.Green, matrix[index].Green);
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(RgbPixel)}");
}
try
{
using (var matrix = new Matrix <RgbAlphaPixel>(1, 3))
for (var index = 0; index < 3; index++)
{
var b = (byte)(index);
var v = new RgbAlphaPixel {
Red = b, Blue = b, Green = b
};
matrix[index] = v;
Assert.AreEqual(v.Red, matrix[index].Red);
Assert.AreEqual(v.Blue, matrix[index].Blue);
Assert.AreEqual(v.Green, matrix[index].Green);
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(RgbPixel)}");
}
try
{
using (var matrix = new Matrix <HsiPixel>(1, 3))
for (var index = 0; index < 3; index++)
{
var b = (byte)(index);
var v = new HsiPixel {
H = b, S = b, I = b
};
matrix[index] = v;
Assert.AreEqual(v.H, matrix[index].H);
Assert.AreEqual(v.S, matrix[index].S);
Assert.AreEqual(v.I, matrix[index].I);
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(HsiPixel)}");
}
}
public Voc2012Class(ushort index, RgbPixel rgbLabel, string classLabel)
{
this.Index = index;
this.RgbLabel = rgbLabel;
this.ClassLabel = classLabel;
}
public static void CreateFeatureVectors()
{
int faceCount = 0;
float leftEyebrow, rightEyebrow, leftLip, rightLip, lipHeight, lipWidth;
string output;
if (currentDataType == Datatype.Testing)
{
output = testingOutput;
}
else
{
output = trainingOutput;
}
string[] dirs = Directory.GetFiles(currentFilePath, "*.*", SearchOption.AllDirectories);
// Set up Dlib Face Detector
using (var fd = Dlib.GetFrontalFaceDetector())
// ... and Dlib Shape Detector
using (var sp = ShapePredictor.Deserialize("shape_predictor_68_face_landmarks.dat"))
{
string header = "leftEyebrow,rightEyebrow,leftLip,rightLip,lipWidth,lipHeight,label\n";
// Create the CSV file and fill in the first line with the header
System.IO.File.WriteAllText(output, header);
foreach (string dir in dirs)
{
// call function that sets the label based on what the filename contains
string label = DetermineLabel(dir);
// load input image
if (!(dir.EndsWith("png") || dir.EndsWith("jpg")))
{
continue;
}
var img = Dlib.LoadImage <RgbPixel>(dir);
// find all faces in the image
var faces = fd.Operator(img);
// for each face draw over the facial landmarks
foreach (var face in faces)
{
// Write to the console displaying the progress and current emotion
Form1.SetProgress(faceCount, dirs.Length - 1);
// find the landmark points for this face
var shape = sp.Detect(img, face);
for (var i = 0; i < shape.Parts; i++)
{
RgbPixel colour = new RgbPixel(255, 255, 255);
var point = shape.GetPart((uint)i);
var rect = new DlibDotNet.Rectangle(point);
Dlib.DrawRectangle(img, rect, color: colour, thickness: 2);
}
SetFormImage(img);
leftEyebrow = CalculateLeftEyebrow(shape);
rightEyebrow = CalculateRightEyebrow(shape);
leftLip = CalculateLeftLip(shape);
rightLip = CalculateRightLip(shape);
lipWidth = CalculateLipWidth(shape);
lipHeight = CalculateLipHeight(shape);
using (System.IO.StreamWriter file = new System.IO.StreamWriter(output, true))
{
file.WriteLine(leftEyebrow + "," + rightEyebrow + "," + leftLip + "," + rightLip + "," + lipWidth + "," + lipHeight + "," + label);
}
// Increment count used for console output
faceCount++;
}
}
if (currentDataType == Datatype.Testing)
{
var testDataView = mlContext.Data.LoadFromTextFile <FeatureInputData>(output, hasHeader: true, separatorChar: ',');
GenerateMetrics(testDataView);
}
Form1.HideImage();
}
}
