public RgbAlphaPixel Operator(string str)
{
if (this._Colors.TryGetValue(str, out var value))
{
return(value);
}
var pix = new HsiPixel
{
H = Reverse((byte)this._Colors.Count),
S = byte.MaxValue,
I = 150
};
var result = new RgbAlphaPixel();
Dlib.AssignPixel(ref result, pix);
this._Colors[str] = result;
return(result);
}
public void GetRowColumn()
{
const int width = 150;
const int height = 100;
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
foreach (var test in tests)
{
var array2D = CreateArray2D(test.Type, height, width);
switch (array2D.ImageType)
{
case ImageTypes.UInt8:
{
var array = (Array2D <byte>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<byte> failed");
}
break;
case ImageTypes.UInt16:
{
var array = (Array2D <ushort>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<ushort> failed");
}
break;
case ImageTypes.UInt32:
{
var array = (Array2D <uint>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255u, "Array<uint> failed");
}
break;
case ImageTypes.Int8:
{
var array = (Array2D <sbyte>)array2D;
Dlib.AssignAllPpixels(array, 127);
using (var row = array[0])
Assert.AreEqual(row[0], 127, "Array<sbyte> failed");
}
break;
case ImageTypes.Int16:
{
var array = (Array2D <short>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<short> failed");
}
break;
case ImageTypes.Int32:
{
var array = (Array2D <int>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<int> failed");
}
break;
case ImageTypes.Float:
{
var array = (Array2D <float>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<float> failed");
}
break;
case ImageTypes.Double:
{
var array = (Array2D <double>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<double> failed");
}
break;
case ImageTypes.RgbPixel:
{
var array = (Array2D <RgbPixel>)array2D;
var pixel = new RgbPixel
{
Red = 255,
Blue = 255,
Green = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.Red, 255, "Array<RgbPixel> failed");
Assert.AreEqual(t.Blue, 255, "Array<RgbPixel> failed");
Assert.AreEqual(t.Green, 255, "Array<RgbPixel> failed");
}
}
break;
case ImageTypes.RgbAlphaPixel:
{
var array = (Array2D <RgbAlphaPixel>)array2D;
var pixel = new RgbAlphaPixel
{
Red = 255,
Blue = 255,
Green = 255,
Alpha = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.Red, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Blue, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Green, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Alpha, 255, "Array<RgbAlphaPixel> failed");
}
}
break;
case ImageTypes.HsiPixel:
{
var array = (Array2D <HsiPixel>)array2D;
var pixel = new HsiPixel
{
H = 255,
S = 255,
I = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.H, 255, "Array<HsiPixel> failed");
Assert.AreEqual(t.S, 255, "Array<HsiPixel> failed");
Assert.AreEqual(t.I, 255, "Array<HsiPixel> failed");
}
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(array2D.ImageType), array2D.ImageType, null);
}
this.DisposeAndCheckDisposedState(array2D);
}
}
public void LoadImageData2()
{
const int cols = 512;
const int rows = 512;
const int steps = 512;
var tests = new[]
{
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
var random = new Random(0);
foreach (var test in tests)
{
TwoDimentionObjectBase image;
using (var win = new ImageWindow())
{
switch (test.Type)
{
case ImageTypes.UInt8:
{
var data = new byte[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (byte)random.Next(0, 255);
}
}
image = Dlib.LoadImageData <byte>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <byte>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.UInt16:
{
var data = new ushort[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (ushort)random.Next(0, 255);
}
}
image = Dlib.LoadImageData <ushort>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <ushort>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Int16:
{
var data = new short[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (short)random.Next(0, 255);
}
}
image = Dlib.LoadImageData <short>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <short>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Int32:
{
var data = new int[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = random.Next(0, 255);
}
}
image = Dlib.LoadImageData <int>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <int>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Float:
{
var data = new float[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (float)random.NextDouble();
}
}
image = Dlib.LoadImageData <float>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <float>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.Double:
{
var data = new double[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = (double)random.NextDouble();
}
}
image = Dlib.LoadImageData <double>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <double>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.HsiPixel:
{
var data = new HsiPixel[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = new HsiPixel
{
H = (byte)random.Next(0, 255),
S = (byte)random.Next(0, 255),
I = (byte)random.Next(0, 255)
}
}
}
;
image = Dlib.LoadImageData <HsiPixel>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <HsiPixel>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.RgbPixel:
{
var data = new RgbPixel[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = new RgbPixel
{
Red = (byte)random.Next(0, 255),
Green = (byte)random.Next(0, 255),
Blue = (byte)random.Next(0, 255)
}
}
}
;
image = Dlib.LoadImageData <RgbPixel>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <RgbPixel>)image);
win.WaitUntilClosed();
}
}
break;
case ImageTypes.RgbAlphaPixel:
{
var data = new RgbAlphaPixel[rows * cols];
for (var r = 0; r < rows; r++)
{
for (var c = 0; c < cols; c++)
{
data[steps * r + c] = new RgbAlphaPixel
{
Red = (byte)random.Next(0, 255),
Green = (byte)random.Next(0, 255),
Blue = (byte)random.Next(0, 255),
Alpha = (byte)random.Next(0, 255)
}
}
}
;
image = Dlib.LoadImageData <RgbAlphaPixel>(data, rows, cols, steps);
if (this.CanGuiDebug)
{
win.SetImage((Array2D <RgbAlphaPixel>)image);
win.WaitUntilClosed();
}
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
Assert.AreEqual(image.Columns, cols, $"Failed to load {test.Type}.");
Assert.AreEqual(image.Rows, rows, $"Failed to load {test.Type}.");
this.DisposeAndCheckDisposedState(image);
}
}
public void Indexer3()
{
try
{
using (var matrix = new Matrix <byte>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (byte)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(byte)}");
}
try
{
using (var matrix = new Matrix <ushort>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (ushort)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(ushort)}");
}
try
{
using (var matrix = new Matrix <uint>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (uint)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(uint)}");
}
try
{
using (var matrix = new Matrix <sbyte>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (sbyte)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(sbyte)}");
}
try
{
using (var matrix = new Matrix <short>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (short)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(short)}");
}
try
{
using (var matrix = new Matrix <int>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (int)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(int)}");
}
try
{
using (var matrix = new Matrix <float>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (float)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(float)}");
}
try
{
using (var matrix = new Matrix <double>(1, 3))
{
for (var index = 0; index < 3; index++)
{
var v = (double)(index);
matrix[index] = v;
Assert.AreEqual(v, matrix[index]);
}
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(double)}");
}
try
{
using (var matrix = new Matrix <RgbPixel>(1, 3))
for (var index = 0; index < 3; index++)
{
var b = (byte)(index);
var v = new RgbPixel {
Red = b, Blue = b, Green = b
};
matrix[index] = v;
Assert.AreEqual(v.Red, matrix[index].Red);
Assert.AreEqual(v.Blue, matrix[index].Blue);
Assert.AreEqual(v.Green, matrix[index].Green);
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(RgbPixel)}");
}
try
{
using (var matrix = new Matrix <RgbAlphaPixel>(1, 3))
for (var index = 0; index < 3; index++)
{
var b = (byte)(index);
var v = new RgbAlphaPixel {
Red = b, Blue = b, Green = b
};
matrix[index] = v;
Assert.AreEqual(v.Red, matrix[index].Red);
Assert.AreEqual(v.Blue, matrix[index].Blue);
Assert.AreEqual(v.Green, matrix[index].Green);
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(RgbPixel)}");
}
try
{
using (var matrix = new Matrix <HsiPixel>(1, 3))
for (var index = 0; index < 3; index++)
{
var b = (byte)(index);
var v = new HsiPixel {
H = b, S = b, I = b
};
matrix[index] = v;
Assert.AreEqual(v.H, matrix[index].H);
Assert.AreEqual(v.S, matrix[index].S);
Assert.AreEqual(v.I, matrix[index].I);
}
}
catch (Exception)
{
Assert.Fail($"Failed to access for Type: {typeof(HsiPixel)}");
}
}
private static void Main()
{
try
{
// You can get this file from http://dlib.net/files/mmod_rear_end_vehicle_detector.dat.bz2
// This network was produced by the dnn_mmod_train_find_cars_ex.cpp example program.
// As you can see, the file also includes a separately trained shape_predictor. To see
// a generic example of how to train those refer to train_shape_predictor_ex.cpp.
using (var deserialize = new ProxyDeserialize("mmod_rear_end_vehicle_detector.dat"))
using (var net = LossMmod.Deserialize(deserialize, 1))
using (var sp = ShapePredictor.Deserialize(deserialize))
using (var img = Dlib.LoadImageAsMatrix <RgbPixel>("mmod_cars_test_image.jpg"))
using (var win = new ImageWindow())
{
win.SetImage(img);
// Run the detector on the image and show us the output.
var dets = net.Operator(img).First();
foreach (var d in dets)
{
// We use a shape_predictor to refine the exact shape and location of the detection
// box. This shape_predictor is trained to simply output the 4 corner points of
// the box. So all we do is make a rectangle that tightly contains those 4 points
// and that rectangle is our refined detection position.
var fd = sp.Detect(img, d);
var rect = Rectangle.Empty;
for (var j = 0u; j < fd.Parts; ++j)
{
rect += fd.GetPart(j);
}
win.AddOverlay(rect, new RgbPixel(255, 0, 0));
}
Console.WriteLine("Hit enter to view the intermediate processing steps");
Console.ReadKey();
// Now let's look at how the detector works. The high level processing steps look like:
// 1. Create an image pyramid and pack the pyramid into one big image. We call this
// image the "tiled pyramid".
// 2. Run the tiled pyramid image through the CNN. The CNN outputs a new image where
// bright pixels in the output image indicate the presence of cars.
// 3. Find pixels in the CNN's output image with a value > 0. Those locations are your
// preliminary car detections.
// 4. Perform non-maximum suppression on the preliminary detections to produce the
// final output.
//
// We will be plotting the images from steps 1 and 2 so you can visualize what's
// happening. For the CNN's output image, we will use the jet colormap so that "bright"
// outputs, i.e. pixels with big values, appear in red and "dim" outputs appear as a
// cold blue color. To do this we pick a range of CNN output values for the color
// mapping. The specific values don't matter. They are just selected to give a nice
// looking output image.
const float lower = -2.5f;
const float upper = 0.0f;
Console.WriteLine($"jet color mapping range: lower={lower} upper={upper}");
// Create a tiled pyramid image and display it on the screen.
// Get the type of pyramid the CNN used
//using pyramid_type = std::remove_reference < decltype(input_layer(net)) >::type::pyramid_type;
// And tell create_tiled_pyramid to create the pyramid using that pyramid type.
using (var inputLayer = new InputRgbImagePyramid <PyramidDown>(6))
{
net.TryGetInputLayer(inputLayer);
var padding = inputLayer.GetPyramidPadding();
var outerPadding = inputLayer.GetPyramidOuterPadding();
Dlib.CreateTiledPyramid <RgbPixel, PyramidDown>(img,
padding,
outerPadding,
6,
out var tiledImg,
out var rects);
using (var winpyr = new ImageWindow(tiledImg, "Tiled pyramid"))
{
// This CNN detector represents a sliding window detector with 3 sliding windows. Each
// of the 3 windows has a different aspect ratio, allowing it to find vehicles which
// are either tall and skinny, squarish, or short and wide. The aspect ratio of a
// detection is determined by which channel in the output image triggers the detection.
// Here we are just going to max pool the channels together to get one final image for
// our display. In this image, a pixel will be bright if any of the sliding window
// detectors thinks there is a car at that location.
using (var subnet = net.GetSubnet())
{
var output = subnet.Output;
Console.WriteLine($"Number of channels in final tensor image: {output.K}");
var networkOutput = Dlib.ImagePlane(output);
for (var k = 1; k < output.K; k++)
{
using (var tmpNetworkOutput = Dlib.ImagePlane(output, 0, k))
{
var maxPointWise = Dlib.MaxPointWise(networkOutput, tmpNetworkOutput);
networkOutput.Dispose();
networkOutput = maxPointWise;
}
}
// We will also upsample the CNN's output image. The CNN we defined has an 8x
// downsampling layer at the beginning. In the code below we are going to overlay this
// CNN output image on top of the raw input image. To make that look nice it helps to
// upsample the CNN output image back to the same resolution as the input image, which
// we do here.
var networkOutputScale = img.Columns / (double)networkOutput.Columns;
Dlib.ResizeImage(networkOutput, networkOutputScale);
// Display the network's output as a color image.
using (var jet = Dlib.Jet(networkOutput, upper, lower))
using (var winOutput = new ImageWindow(jet, "Output tensor from the network"))
{
// Also, overlay network_output on top of the tiled image pyramid and display it.
for (var r = 0; r < tiledImg.Rows; ++r)
{
for (var c = 0; c < tiledImg.Columns; ++c)
{
var tmp = new DPoint(c, r);
tmp = Dlib.InputTensorToOutputTensor(net, tmp);
var dp = networkOutputScale * tmp;
tmp = new DPoint((int)dp.X, (int)dp.Y);
if (Dlib.GetRect(networkOutput).Contains((int)tmp.X, (int)tmp.Y))
{
var val = networkOutput[(int)tmp.Y, (int)tmp.X];
// alpha blend the network output pixel with the RGB image to make our
// overlay.
var p = new RgbAlphaPixel();
Dlib.AssignPixel(ref p, Dlib.ColormapJet(val, lower, upper));
p.Alpha = 120;
var rgb = new RgbPixel();
Dlib.AssignPixel(ref rgb, p);
tiledImg[r, c] = rgb;
}
}
}
// If you look at this image you can see that the vehicles have bright red blobs on
// them. That's the CNN saying "there is a car here!". You will also notice there is
// a certain scale at which it finds cars. They have to be not too big or too small,
// which is why we have an image pyramid. The pyramid allows us to find cars of all
// scales.
using (var winPyrOverlay = new ImageWindow(tiledImg, "Detection scores on image pyramid"))
{
// Finally, we can collapse the pyramid back into the original image. The CNN doesn't
// actually do this step, since it's enough to threshold the tiled pyramid image to get
// the detections. However, it makes a nice visualization and clearly indicates that
// the detector is firing for all the cars.
using (var collapsed = new Matrix <float>(img.Rows, img.Columns))
using (var inputTensor = new ResizableTensor())
{
inputLayer.ToTensor(img, 1, inputTensor);
for (var r = 0; r < collapsed.Rows; ++r)
{
for (var c = 0; c < collapsed.Columns; ++c)
{
// Loop over a bunch of scale values and look up what part of network_output
// corresponds to the point(c,r) in the original image, then take the max
// detection score over all the scales and save it at pixel point(c,r).
var maxScore = -1e30f;
for (double scale = 1; scale > 0.2; scale *= 5.0 / 6.0)
{
// Map from input image coordinates to tiled pyramid coordinates.
var tensorSpace = inputLayer.ImageSpaceToTensorSpace(inputTensor, scale, new DRectangle(new DPoint(c, r)));
var tmp = tensorSpace.Center;
// Now map from pyramid coordinates to network_output coordinates.
var dp = networkOutputScale * Dlib.InputTensorToOutputTensor(net, tmp);
tmp = new DPoint((int)dp.X, (int)dp.Y);
if (Dlib.GetRect(networkOutput).Contains((int)tmp.X, (int)tmp.Y))
{
var val = networkOutput[(int)tmp.Y, (int)tmp.X];
if (val > maxScore)
{
maxScore = val;
}
}
}
collapsed[r, c] = maxScore;
// Also blend the scores into the original input image so we can view it as
// an overlay on the cars.
var p = new RgbAlphaPixel();
Dlib.AssignPixel(ref p, Dlib.ColormapJet(maxScore, lower, upper));
p.Alpha = 120;
var rgb = new RgbPixel();
Dlib.AssignPixel(ref rgb, p);
img[r, c] = rgb;
}
}
using (var jet2 = Dlib.Jet(collapsed, upper, lower))
using (var winCollapsed = new ImageWindow(jet2, "Collapsed output tensor from the network"))
using (var winImgAndSal = new ImageWindow(img, "Collapsed detection scores on raw image"))
{
Console.WriteLine("Hit enter to end program");
Console.ReadKey();
}
}
}
}
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
