public BaseResponse <string> UploadUserAvatar()
{
var avatarFile = Request.Form.Files[0];
var userToken = Request.Form["userToken"][0];
if (string.IsNullOrEmpty(userToken))
{
return new BaseResponse <string> {
Success = false, Message = "User token not sent"
}
}
;
if (avatarFile == null)
{
return new BaseResponse <string> {
Success = false, Message = "Avatar not sent"
}
}
;
var user = UserService.GetUser(userToken);
if (user != null)
{
var fileExtension = "png";
if (avatarFile.ContentType.IndexOf("jpeg") != -1)
{
fileExtension = "jpeg";
}
else if (avatarFile.ContentType.IndexOf("jpg") != -1)
{
fileExtension = "jpg";
}
var filename = Helper.Util.GenerateFileName(user.UserID, fileExtension, "avatar_f_");
if (UserService.UpdateAvatar(user.UserID, filename))
{
var uploads = Path.Combine("wwwroot/avatars", filename);
avatarFile.CopyTo(new FileStream(uploads, FileMode.Create));
using (var fd = Dlib.GetFrontalFaceDetector())
{
var img = Dlib.LoadImage <RgbPixel>(uploads);
// find all faces in the image
var faces = fd.Operator(img);
foreach (var face in faces)
{
// draw a rectangle for each face
Dlib.DrawRectangle(img, face, color: new RgbPixel(0, 255, 255), thickness: 4);
}
uploads = uploads.Replace("avatar_f_", "avatar_");
Dlib.SaveJpeg(img, uploads);
}
return(new BaseResponse <string> {
Data = filename, Success = true
});
}
else
{
return new BaseResponse <string> {
Success = false, Message = "Error on update avatar"
}
};
}
return(new BaseResponse <string> {
Success = false, Message = "Token not find a user"
});
}
private static void Main(string[] args)
{
var x = new List <string>( );
x.Add(@"shape_predictor_68_face_landmarks.dat");
args = x.ToArray();
if (args.Length == 0)
{
Console.WriteLine("Give some image files as arguments to this program.");
Console.WriteLine("Call this program like this:");
Console.WriteLine("./face_landmark_detection_ex shape_predictor_68_face_landmarks.dat faces/*.jpg");
Console.WriteLine("You can get the shape_predictor_68_face_landmarks.dat file from:");
Console.WriteLine("http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2");
return;
}
using (var win = new ImageWindow())
using (var winFaces = new ImageWindow())
using (var detector = Dlib.GetFrontalFaceDetector())
using (var sp = ShapePredictor.Deserialize(args[0]))
foreach (var file in args.ToList().GetRange(1, args.Length - 1))
{
Console.WriteLine($"processing image {file}");
using (var img = Dlib.LoadImage <RgbPixel>(file))
{
Dlib.PyramidUp(img);
var dets = detector.Detect(img);
Console.WriteLine($"Number of faces detected: {dets.Length}");
var shapes = new List <FullObjectDetection>();
foreach (var rect in dets)
{
var shape = sp.Detect(img, rect);
Console.WriteLine($"number of parts: {shape.Parts}");
if (shape.Parts > 2)
{
Console.WriteLine($"pixel position of first part: {shape.GetPart(0)}");
Console.WriteLine($"pixel position of second part: {shape.GetPart(1)}");
shapes.Add(shape);
}
}
win.ClearOverlay();
win.SetImage(img);
if (shapes.Any())
{
var lines = Dlib.RenderFaceDetections(shapes);
win.AddOverlay(lines);
foreach (var l in lines)
{
l.Dispose();
}
var chipLocations = Dlib.GetFaceChipDetails(shapes);
using (var faceChips = Dlib.ExtractImageChips <RgbPixel>(img, chipLocations))
using (var tileImage = Dlib.TileImages(faceChips))
winFaces.SetImage(tileImage);
foreach (var c in chipLocations)
{
c.Dispose();
}
}
Console.WriteLine("hit enter to process next frame");
Console.ReadKey();
foreach (var s in shapes)
{
s.Dispose();
}
}
}
}
public void CheckSize()
{
const int width = 150;
const int height = 100;
const string testName = "CreateWithSize";
var tests = new[]
{
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true },
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true }
};
foreach (var output in tests)
{
var expectResult = output.ExpectResult;
var outputImageAction = new Func <bool, Array2DBase>(expect => Array2DTest.CreateArray2D(output.Type, height, width));
var successAction = new Action <Array2DBase>(array =>
{
try
{
using (var mat = Dlib.Mat(array))
{
Assert.AreEqual(mat.Columns, width, $"Columns should equal to {width}.");
Assert.AreEqual(mat.Rows, height, $"Rows should equal to {height}.");
}
}
finally
{
if (array != null)
{
this.DisposeAndCheckDisposedState(array);
}
}
});
var failAction = new Action(() =>
{
Assert.Fail($"{testName} should throw exception for InputType: {output.Type}.");
});
var finallyAction = new Action(() =>
{
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {output.Type}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
public void RotateImage2()
{
const string testName = "RotateImage2";
var path = this.GetDataFile($"{LoadTarget}.bmp");
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
var type = this.GetType().Name;
foreach (InterpolationTypes itype in Enum.GetValues(typeof(InterpolationTypes)))
{
foreach (var angle in new[] { 30, 60, 90 })
{
foreach (var input in tests)
{
var expectResult = input.ExpectResult;
var imageObj = DlibTest.LoadImage(input.Type, path);
var outputObj = Array2DTest.CreateArray2D(input.Type);
var outputImageAction = new Func <bool, Array2DBase>(expect =>
{
Dlib.RotateImage(imageObj, outputObj, angle, itype);
return(outputObj);
});
var successAction = new Action <Array2DBase>(image =>
{
Dlib.SaveJpeg(outputObj, $"{Path.Combine(this.GetOutDir(type, testName), $"{LoadTarget}_{input.Type}_{input.Type}_{angle}_{itype}.jpg")}");
});
var failAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {input.Type}, OutputType: {input.Type}, Angle: {angle}, InterpolationType: {itype}.");
});
var finallyAction = new Action(() =>
{
if (imageObj != null)
{
this.DisposeAndCheckDisposedState(imageObj);
}
if (outputObj != null)
{
this.DisposeAndCheckDisposedState(outputObj);
}
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {input.Type}, OutputType: {input.Type}, Angle: {angle}, InterpolationType: {itype}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
}
}
public void TransformImagePointTransformProjective()
{
const string testName = "TransformImagePointTransformProjective";
var path = this.GetDataFile($"{LoadTarget}.bmp");
var tests = new[]
{
new { Type = ImageTypes.HsiPixel, ExpectResult = false },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = false },
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
var angleCases = new[]
{
//45d, -45d, 90d, 180d
0d
};
var xCases = new[]
{
//10, -10, 50,5, -50,5
0d
};
var yCases = new[]
{
//10, -10, 50,5, -50,5
0d
};
var type = this.GetType().Name;
foreach (var angle in angleCases)
{
foreach (var x in xCases)
{
foreach (var y in yCases)
{
foreach (var input in tests)
{
foreach (var output in tests)
{
var expectResult = input.ExpectResult && output.ExpectResult;
var imageObj = DlibTest.LoadImage(input.Type, path);
var outputObj = Array2DTest.CreateArray2D(output.Type, imageObj.Rows, imageObj.Columns);
var matrix = new Matrix <double>(3, 3);
var transform = new PointTransformProjective(matrix);
var outputImageAction = new Func <bool, Array2DBase>(expect =>
{
Dlib.TransformImage(imageObj, outputObj, transform);
return(outputObj);
});
var successAction = new Action <Array2DBase>(image =>
{
Dlib.SaveJpeg(image, $"{Path.Combine(this.GetOutDir(type, testName), $"{LoadTarget}_{input.Type}_{output.Type}_{angle}_{x}_{y}.jpg")}");
});
var failAction = new Action(() =>
{
Assert.Fail($"{testName} should throw exception for InputType: {input.Type}, OutputType: {output.Type}, Angle,: {angle}, X: {x}, Y: {y}.");
});
var finallyAction = new Action(() =>
{
this.DisposeAndCheckDisposedState(matrix);
this.DisposeAndCheckDisposedState(transform);
this.DisposeAndCheckDisposedState(imageObj);
if (outputObj != null)
{
this.DisposeAndCheckDisposedState(outputObj);
}
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {input.Type}, OutputType: {output.Type}, Angle,: {angle}, X: {x}, Y: {y}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
}
}
}
}
public JsonResult CreatePost()
{
try
{
var photo = Request.Files[0];
var token = Request.Form["userToken"];
var description = Request.Form["description"];
var category = (Category)int.Parse(Request.Form["category"]);
PostViewModel postViewModel = new PostViewModel()
{
Description = description, Token = token, Category = category
};
var user = UserService.GetUser(postViewModel.Token);
if (user != null && photo != null)
{
var fileExtension = "png";
if (photo.ContentType.IndexOf("jpeg") != -1)
{
fileExtension = "jpeg";
}
else if (photo.ContentType.IndexOf("jpg") != -1)
{
fileExtension = "jpg";
}
var filename = Helper.Util.GenerateFileName(user.UserID, fileExtension, "post_f_");
var uploads = Path.Combine(System.Configuration.ConfigurationManager.AppSettings["MediaPath"] + "Posts", filename);
photo.SaveAs(uploads);
using (var fd = Dlib.GetFrontalFaceDetector())
{
var img = Dlib.LoadImage <RgbPixel>(uploads);
// find all faces in the image
var faces = fd.Operator(img);
uploads = uploads.Replace("post_f_", "post_");
filename = filename.Replace("post_f_", "post_");
//bool needSaveByDlib = true;
foreach (var face in faces)
{
// draw a rectangle for each face
Dlib.DrawRectangle(img, face, color: new RgbPixel(0, 0, 0), thickness: 10);
Dlib.FillRect(img, face, new RgbPixel(0, 0, 0));
//var image = Bitmap.FromFile(uploads);
/*using (var stream = new MemoryStream())
* {
* photo.CopyTo(stream);
* using (var img2 = System.Drawing.Image.FromStream(stream))
* {
* using (var graphic = Graphics.FromImage(img2))
* {
* //var font = new Font(FontFamily.GenericSansSerif, 20, FontStyle.Bold, GraphicsUnit.Pixel);
* //var color = Color.FromArgb(128, 255, 255, 255);
* //var brush = new SolidBrush(color);
* //var point = new System.Drawing.Point(img2.Width - 120, img2.Height - 30);
*
* //graphic.DrawString("edi.wang", font, brush, point);
* using (var imgEmotion = System.Drawing.Image.FromFile(Path.Combine(System.Configuration.ConfigurationManager.AppSettings["MediaPath"] + "images", "emotion.png")))
* graphic.DrawImage(imgEmotion, new RectangleF(face.Left, face.Top, face.Width, face.Height));
*
* img2.Save(uploads, System.Drawing.Imaging.ImageFormat.Png);
* needSaveByDlib = false;
* }
* }
* }*/
}
//if (needSaveByDlib)
Dlib.SaveJpeg(img, uploads);
}
PostService.Insert(new Post()
{
Category = postViewModel.Category,
CreationDate = DateTime.Now,
Description = postViewModel.Description,
Filename = filename,
Comments = 0,
Likes = 0,
Username = user.Username,
UserID = user.UserID
});
return(Json(new BaseResponse <bool>()
{
Data = true, Message = "OK", Success = true
}));
}
else
{
return(Json(new BaseResponse <bool>()
{
Data = false, Message = "User not found!", Success = false
}));
}
}
catch (Exception ex)
{
return(Json(new BaseResponse <bool>()
{
Data = false, Message = "Operation fail, try again!<!--" + ex.Message + "-->", Success = false
}));
}
}
private void ExecuteMaxCostAssignment(TwoDimensionObjectBase obj)
{
if (obj is Matrix <sbyte> sbyteMatrix)
{
Dlib.MaxCostAssignment(sbyteMatrix);
return;
}
if (obj is Matrix <short> shortMatrix)
{
Dlib.MaxCostAssignment(shortMatrix);
return;
}
if (obj is Matrix <int> intMatrix)
{
Dlib.MaxCostAssignment(intMatrix);
return;
}
if (obj is Matrix <byte> byteMatrix)
{
Dlib.MaxCostAssignment(byteMatrix);
return;
}
if (obj is Matrix <ushort> ushortMatrix)
{
Dlib.MaxCostAssignment(ushortMatrix);
return;
}
if (obj is Matrix <uint> uintMatrix)
{
Dlib.MaxCostAssignment(uintMatrix);
return;
}
if (obj is Matrix <float> floatMatrix)
{
Dlib.MaxCostAssignment(floatMatrix);
return;
}
if (obj is Matrix <double> doubleMatrix)
{
Dlib.MaxCostAssignment(doubleMatrix);
return;
}
if (obj is Matrix <RgbPixel> rgbPixelMatrix)
{
Dlib.MaxCostAssignment(rgbPixelMatrix);
return;
}
if (obj is Matrix <RgbAlphaPixel> rgbAlphaPixelMatrix)
{
Dlib.MaxCostAssignment(rgbAlphaPixelMatrix);
return;
}
if (obj is Matrix <HsiPixel> hsiPicelMatrix)
{
Dlib.MaxCostAssignment(hsiPicelMatrix);
return;
}
throw new NotSupportedException();
}
public void GetRowColumn()
{
const int width = 150;
const int height = 100;
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
foreach (var test in tests)
{
var array2D = CreateArray2D(test.Type, height, width);
switch (array2D.ImageType)
{
case ImageTypes.UInt8:
{
var array = (Array2D <byte>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<byte> failed");
}
break;
case ImageTypes.UInt16:
{
var array = (Array2D <ushort>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<ushort> failed");
}
break;
case ImageTypes.Int16:
{
var array = (Array2D <short>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<short> failed");
}
break;
case ImageTypes.Int32:
{
var array = (Array2D <int>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<int> failed");
}
break;
case ImageTypes.Float:
{
var array = (Array2D <float>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<float> failed");
}
break;
case ImageTypes.Double:
{
var array = (Array2D <double>)array2D;
Dlib.AssignAllPpixels(array, 255);
using (var row = array[0])
Assert.AreEqual(row[0], 255, "Array<double> failed");
}
break;
case ImageTypes.RgbPixel:
{
var array = (Array2D <RgbPixel>)array2D;
var pixel = new RgbPixel
{
Red = 255,
Blue = 255,
Green = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.Red, 255, "Array<RgbPixel> failed");
Assert.AreEqual(t.Blue, 255, "Array<RgbPixel> failed");
Assert.AreEqual(t.Green, 255, "Array<RgbPixel> failed");
}
}
break;
case ImageTypes.RgbAlphaPixel:
{
var array = (Array2D <RgbAlphaPixel>)array2D;
var pixel = new RgbAlphaPixel
{
Red = 255,
Blue = 255,
Green = 255,
Alpha = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.Red, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Blue, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Green, 255, "Array<RgbAlphaPixel> failed");
Assert.AreEqual(t.Alpha, 255, "Array<RgbAlphaPixel> failed");
}
}
break;
case ImageTypes.HsiPixel:
{
var array = (Array2D <HsiPixel>)array2D;
var pixel = new HsiPixel
{
H = 255,
S = 255,
I = 255
};
Dlib.AssignAllPpixels(array, pixel);
using (var row = array[0])
{
var t = row[0];
Assert.AreEqual(t.H, 255, "Array<HsiPixel> failed");
Assert.AreEqual(t.S, 255, "Array<HsiPixel> failed");
Assert.AreEqual(t.I, 255, "Array<HsiPixel> failed");
}
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(array2D.ImageType), array2D.ImageType, null);
}
this.DisposeAndCheckDisposedState(array2D);
}
}
// the C# spec says that one class cannot call another's constructor. Thank goodness this is not the case in VB.Net.
// https://stackoverflow.com/questions/19162656/why-is-this-c-sharp-constructor-not-working-as-expected/19162779
public void initialize()
{
detector = Dlib.GetFrontalFaceDetector();
}
private static void Main()
{
using (var img = new Array2D <byte>(400, 400))
using (var ht = new DlibDotNet.HoughTransform(300))
using (var win = new ImageWindow())
using (var win2 = new ImageWindow())
{
var angle1 = 0d;
var angle2 = 0d;
while (true)
{
angle1 += Math.PI / 130;
angle2 += Math.PI / 400;
using (var rect = img.Rect)
using (var cent = rect.Center)
using (var point1 = new Point(90, 0))
using (var tmp1 = cent + point1)
using (var arc = Point.Rotate(cent, tmp1, angle1 * 180 / Math.PI))
using (var point2 = new Point(500, 0))
using (var tmp2 = arc + point2)
using (var tmp3 = arc - point2)
using (var l = Point.Rotate(arc, tmp2, angle2 * 180 / Math.PI))
using (var r = Point.Rotate(arc, tmp3, angle2 * 180 / Math.PI))
{
Dlib.AssignAllPpixels(img, 0);
Dlib.DrawLine(img, l, r, 255);
using (var offset = new Point(50, 50))
using (var himg = new Array2D <int>())
using (var hrect = Dlib.GetRect(ht))
using (var box = Rectangle.Translate(hrect, offset))
{
// Now let's compute the hough transform for a subwindow in the image. In
// particular, we run it on the 300x300 subwindow with an upper left corner at the
// pixel point(50,50). The output is stored in himg.
ht.Operator(img, box, himg);
// Now that we have the transformed image, the Hough image pixel with the largest
// value should indicate where the line is. So we find the coordinates of the
// largest pixel:
using (var mat = Dlib.Mat(himg))
using (var p = Dlib.MaxPoint(mat))
{
// And then ask the ht object for the line segment in the original image that
// corresponds to this point in Hough transform space.
using (var line = ht.GetLine(p))
{
// Finally, let's display all these things on the screen. We copy the original
// input image into a color image and then draw the detected line on top in red.
using (var temp = new Array2D <RgbPixel>())
{
Dlib.AssignImage(img, temp);
using (var p1 = line.First + offset)
using (var p2 = line.Second + offset)
{
Dlib.DrawLine(temp, p1, p2, new RgbPixel
{
Red = 255
});
win.ClearOverlay();
win.SetImage(temp);
// Also show the subwindow we ran the Hough transform on as a green box. You will
// see that the detected line is exactly contained within this box and also
// overlaps the original line.
win.AddOverlay(box, new RgbPixel
{
Green = 255
});
using (var jet = Dlib.Jet(himg))
win2.SetImage(jet);
}
}
}
}
}
}
}
}
}
private static void Main(string[] args)
{
try
{
if (args.Length != 1)
{
Console.WriteLine("Give the path to a folder containing training.xml and testing.xml files.");
Console.WriteLine("This example program is specifically designed to run on the dlib vehicle ");
Console.WriteLine("detection dataset, which is available at this URL: ");
Console.WriteLine(" http://dlib.net/files/data/dlib_rear_end_vehicles_v1.tar");
Console.WriteLine();
Console.WriteLine("So download that dataset, extract it somewhere, and then run this program");
Console.WriteLine("with the dlib_rear_end_vehicles folder as an argument. E.g. if you extract");
Console.WriteLine("the dataset to the current folder then you should run this example program");
Console.WriteLine("by typing: ");
Console.WriteLine(" ./dnn_mmod_train_find_cars_ex dlib_rear_end_vehicles");
Console.WriteLine();
Console.WriteLine("It takes about a day to finish if run on a high end GPU like a 1080ti.");
Console.WriteLine();
return;
}
var dataDirectory = args[0];
IList <Matrix <RgbPixel> > imagesTrain;
IList <Matrix <RgbPixel> > imagesTest;
IList <IList <MModRect> > boxesTrain;
IList <IList <MModRect> > boxesTest;
Dlib.LoadImageDataset(dataDirectory + "/training.xml", out imagesTrain, out boxesTrain);
Dlib.LoadImageDataset(dataDirectory + "/testing.xml", out imagesTest, out boxesTest);
// When I was creating the dlib vehicle detection dataset I had to label all the cars
// in each image. MMOD requires all cars to be labeled, since any unlabeled part of an
// image is implicitly assumed to be not a car, and the algorithm will use it as
// negative training data. So every car must be labeled, either with a normal
// rectangle or an "ignore" rectangle that tells MMOD to simply ignore it (i.e. neither
// treat it as a thing to detect nor as negative training data).
//
// In our present case, many images contain very tiny cars in the distance, ones that
// are essentially just dark smudges. It's not reasonable to expect the CNN
// architecture we defined to detect such vehicles. However, I erred on the side of
// having more complete annotations when creating the dataset. So when I labeled these
// images I labeled many of these really difficult cases as vehicles to detect.
//
// So the first thing we are going to do is clean up our dataset a little bit. In
// particular, we are going to mark boxes smaller than 35*35 pixels as ignore since
// only really small and blurry cars appear at those sizes. We will also mark boxes
// that are heavily overlapped by another box as ignore. We do this because we want to
// allow for stronger non-maximum suppression logic in the learned detector, since that
// will help make it easier to learn a good detector.
//
// To explain this non-max suppression idea further it's important to understand how
// the detector works. Essentially, sliding window detectors scan all image locations
// and ask "is there a car here?". If there really is a car in a specific location in
// an image then usually many slightly different sliding window locations will produce
// high detection scores, indicating that there is a car at those locations. If we
// just stopped there then each car would produce multiple detections. But that isn't
// what we want. We want each car to produce just one detection. So it's common for
// detectors to include "non-maximum suppression" logic which simply takes the
// strongest detection and then deletes all detections "close to" the strongest. This
// is a simple post-processing step that can eliminate duplicate detections. However,
// we have to define what "close to" means. We can do this by looking at your training
// data and checking how close the closest target boxes are to each other, and then
// picking a "close to" measure that doesn't suppress those target boxes but is
// otherwise as tight as possible. This is exactly what the mmod_options object does
// by default.
//
// Importantly, this means that if your training dataset contains an image with two
// target boxes that really overlap a whole lot, then the non-maximum suppression
// "close to" measure will be configured to allow detections to really overlap a whole
// lot. On the other hand, if your dataset didn't contain any overlapped boxes at all,
// then the non-max suppression logic would be configured to filter out any boxes that
// overlapped at all, and thus would be performing a much stronger non-max suppression.
//
// Why does this matter? Well, remember that we want to avoid duplicate detections.
// If non-max suppression just kills everything in a really wide area around a car then
// the CNN doesn't really need to learn anything about avoiding duplicate detections.
// However, if non-max suppression only suppresses a tiny area around each detection
// then the CNN will need to learn to output small detection scores for those areas of
// the image not suppressed. The smaller the non-max suppression region the more the
// CNN has to learn and the more difficult the learning problem will become. This is
// why we remove highly overlapped objects from the training dataset. That is, we do
// it so the non-max suppression logic will be able to be reasonably effective. Here
// we are ensuring that any boxes that are entirely contained by another are
// suppressed. We also ensure that boxes with an intersection over union of 0.5 or
// greater are suppressed. This will improve the resulting detector since it will be
// able to use more aggressive non-max suppression settings.
var numOverlappedIgnoredTest = 0;
foreach (var v in boxesTest)
{
using (var overlap = new TestBoxOverlap(0.50, 0.95))
numOverlappedIgnoredTest += IgnoreOverlappedBoxes(v, overlap);
}
var numOverlappedIgnored = 0;
var numAdditionalIgnored = 0;
foreach (var v in boxesTrain)
{
using (var overlap = new TestBoxOverlap(0.50, 0.95))
numOverlappedIgnored += IgnoreOverlappedBoxes(v, overlap);
foreach (var bb in v)
{
if (bb.Rect.Width < 35 && bb.Rect.Height < 35)
{
if (!bb.Ignore)
{
bb.Ignore = true;
++numAdditionalIgnored;
}
}
// The dlib vehicle detection dataset doesn't contain any detections with
// really extreme aspect ratios. However, some datasets do, often because of
// bad labeling. So it's a good idea to check for that and either eliminate
// those boxes or set them to ignore. Although, this depends on your
// application.
//
// For instance, if your dataset has boxes with an aspect ratio
// of 10 then you should think about what that means for the network
// architecture. Does the receptive field even cover the entirety of the box
// in those cases? Do you care about these boxes? Are they labeling errors?
// I find that many people will download some dataset from the internet and
// just take it as given. They run it through some training algorithm and take
// the dataset as unchallengeable truth. But many datasets are full of
// labeling errors. There are also a lot of datasets that aren't full of
// errors, but are annotated in a sloppy and inconsistent way. Fixing those
// errors and inconsistencies can often greatly improve models trained from
// such data. It's almost always worth the time to try and improve your
// training dataset.
//
// In any case, my point is that there are other types of dataset cleaning you
// could put here. What exactly you need depends on your application. But you
// should carefully consider it and not take your dataset as a given. The work
// of creating a good detector is largely about creating a high quality
// training dataset.
}
}
// When modifying a dataset like this, it's a really good idea to print a log of how
// many boxes you ignored. It's easy to accidentally ignore a huge block of data, so
// you should always look and see that things are doing what you expect.
Console.WriteLine($"num_overlapped_ignored: {numOverlappedIgnored}");
Console.WriteLine($"num_additional_ignored: {numAdditionalIgnored}");
Console.WriteLine($"num_overlapped_ignored_test: {numOverlappedIgnoredTest}");
Console.WriteLine($"num training images: {imagesTrain.Count()}");
Console.WriteLine($"num testing images: {imagesTest.Count()}");
// Our vehicle detection dataset has basically 3 different types of boxes. Square
// boxes, tall and skinny boxes (e.g. semi trucks), and short and wide boxes (e.g.
// sedans). Here we are telling the MMOD algorithm that a vehicle is recognizable as
// long as the longest box side is at least 70 pixels long and the shortest box side is
// at least 30 pixels long. mmod_options will use these parameters to decide how large
// each of the sliding windows needs to be so as to be able to detect all the vehicles.
// Since our dataset has basically these 3 different aspect ratios, it will decide to
// use 3 different sliding windows. This means the final con layer in the network will
// have 3 filters, one for each of these aspect ratios.
//
// Another thing to consider when setting the sliding window size is the "stride" of
// your network. The network we defined above downsamples the image by a factor of 8x
// in the first few layers. So when the sliding windows are scanning the image, they
// are stepping over it with a stride of 8 pixels. If you set the sliding window size
// too small then the stride will become an issue. For instance, if you set the
// sliding window size to 4 pixels, then it means a 4x4 window will be moved by 8
// pixels at a time when scanning. This is obviously a problem since 75% of the image
// won't even be visited by the sliding window. So you need to set the window size to
// be big enough relative to the stride of your network. In our case, the windows are
// at least 30 pixels in length, so being moved by 8 pixel steps is fine.
using (var options = new MModOptions(boxesTrain, 70, 30))
{
// This setting is very important and dataset specific. The vehicle detection dataset
// contains boxes that are marked as "ignore", as we discussed above. Some of them are
// ignored because we set ignore to true in the above code. However, the xml files
// also contained a lot of ignore boxes. Some of them are large boxes that encompass
// large parts of an image and the intention is to have everything inside those boxes
// be ignored. Therefore, we need to tell the MMOD algorithm to do that, which we do
// by setting options.overlaps_ignore appropriately.
//
// But first, we need to understand exactly what this option does. The MMOD loss
// is essentially counting the number of false alarms + missed detections produced by
// the detector for each image. During training, the code is running the detector on
// each image in a mini-batch and looking at its output and counting the number of
// mistakes. The optimizer tries to find parameters settings that minimize the number
// of detector mistakes.
//
// This overlaps_ignore option allows you to tell the loss that some outputs from the
// detector should be totally ignored, as if they never happened. In particular, if a
// detection overlaps a box in the training data with ignore==true then that detection
// is ignored. This overlap is determined by calling
// options.overlaps_ignore(the_detection, the_ignored_training_box). If it returns
// true then that detection is ignored.
//
// You should read the documentation for test_box_overlap, the class type for
// overlaps_ignore for full details. However, the gist is that the default behavior is
// to only consider boxes as overlapping if their intersection over union is > 0.5.
// However, the dlib vehicle detection dataset contains large boxes that are meant to
// mask out large areas of an image. So intersection over union isn't an appropriate
// way to measure "overlaps with box" in this case. We want any box that is contained
// inside one of these big regions to be ignored, even if the detection box is really
// small. So we set overlaps_ignore to behave that way with this line.
options.OverlapsIgnore = new TestBoxOverlap(0.5, 0.95);
using (var net = new LossMmod(options, 3))
{
// The final layer of the network must be a con layer that contains
// options.detector_windows.size() filters. This is because these final filters are
// what perform the final "sliding window" detection in the network. For the dlib
// vehicle dataset, there will be 3 sliding window detectors, so we will be setting
// num_filters to 3 here.
var detectorWindows = options.DetectorWindows.ToArray();
using (var subnet = net.GetSubnet())
using (var details = subnet.GetLayerDetails())
{
details.SetNumFilters(detectorWindows.Length);
using (var trainer = new DnnTrainer <LossMmod>(net))
{
trainer.SetLearningRate(0.1);
trainer.BeVerbose();
// While training, we are going to use early stopping. That is, we will be checking
// how good the detector is performing on our test data and when it stops getting
// better on the test data we will drop the learning rate. We will keep doing that
// until the learning rate is less than 1e-4. These two settings tell the trainer to
// do that. Essentially, we are setting the first argument to infinity, and only the
// test iterations without progress threshold will matter. In particular, it says that
// once we observe 1000 testing mini-batches where the test loss clearly isn't
// decreasing we will lower the learning rate.
trainer.SetIterationsWithoutProgressThreshold(50000);
trainer.SetTestIterationsWithoutProgressThreshold(1000);
const string syncFilename = "mmod_cars_sync";
trainer.SetSynchronizationFile(syncFilename, 5 * 60);
IEnumerable <Matrix <RgbPixel> > mini_batch_samples;
IEnumerable <IEnumerable <MModRect> > mini_batch_labels;
using (var cropper = new RandomCropper())
{
cropper.SetSeed(0);
cropper.SetChipDims(350, 350);
// Usually you want to give the cropper whatever min sizes you passed to the
// mmod_options constructor, or very slightly smaller sizes, which is what we do here.
cropper.SetMinObjectSize(69, 28);
cropper.MaxRotationDegrees = 2;
using (var rnd = new Rand())
{
// Log the training parameters to the console
Console.WriteLine($"{trainer}{cropper}");
var cnt = 1;
// Run the trainer until the learning rate gets small.
while (trainer.GetLearningRate() >= 1e-4)
{
// Every 30 mini-batches we do a testing mini-batch.
if (cnt % 30 != 0 || !imagesTest.Any())
{
cropper.Operator(87, imagesTrain, boxesTrain, out mini_batch_samples, out mini_batch_labels);
// We can also randomly jitter the colors and that often helps a detector
// generalize better to new images.
foreach (var img in mini_batch_samples)
{
Dlib.DisturbColors(img, rnd);
}
// It's a good idea to, at least once, put code here that displays the images
// and boxes the random cropper is generating. You should look at them and
// think about if the output makes sense for your problem. Most of the time
// it will be fine, but sometimes you will realize that the pattern of cropping
// isn't really appropriate for your problem and you will need to make some
// change to how the mini-batches are being generated. Maybe you will tweak
// some of the cropper's settings, or write your own entirely separate code to
// create mini-batches. But either way, if you don't look you will never know.
// An easy way to do this is to create a dlib::image_window to display the
// images and boxes.
LossMmod.TrainOneStep(trainer, mini_batch_samples, mini_batch_labels);
mini_batch_samples.DisposeElement();
mini_batch_labels.DisposeElement();
}
else
{
cropper.Operator(87, imagesTest, boxesTest, out mini_batch_samples, out mini_batch_labels);
// We can also randomly jitter the colors and that often helps a detector
// generalize better to new images.
foreach (var img in mini_batch_samples)
{
Dlib.DisturbColors(img, rnd);
}
LossMmod.TestOneStep(trainer, mini_batch_samples, mini_batch_labels);
mini_batch_samples.DisposeElement();
mini_batch_labels.DisposeElement();
}
++cnt;
}
// wait for training threads to stop
trainer.GetNet();
Console.WriteLine("done training");
// Save the network to disk
net.Clean();
LossMmod.Serialize(net, "mmod_rear_end_vehicle_detector.dat");
// It's a really good idea to print the training parameters. This is because you will
// invariably be running multiple rounds of training and should be logging the output
// to a file. This print statement will include many of the training parameters in
// your log.
Console.WriteLine($"{trainer}{cropper}");
Console.WriteLine($"\nsync_filename: {syncFilename}");
Console.WriteLine($"num training images: {imagesTrain.Count()}");
using (var _ = new TestBoxOverlap())
using (var matrix = Dlib.TestObjectDetectionFunction(net, imagesTrain, boxesTrain, _, 0, options.OverlapsIgnore))
Console.WriteLine($"training results: {matrix}");
// Upsampling the data will allow the detector to find smaller cars. Recall that
// we configured it to use a sliding window nominally 70 pixels in size. So upsampling
// here will let it find things nominally 35 pixels in size. Although we include a
// limit of 1800*1800 here which means "don't upsample an image if it's already larger
// than 1800*1800". We do this so we don't run out of RAM, which is a concern because
// some of the images in the dlib vehicle dataset are really high resolution.
Dlib.UpsampleImageDataset(2, imagesTrain, boxesTrain, 1800 * 1800);
using (var _ = new TestBoxOverlap())
using (var matrix = Dlib.TestObjectDetectionFunction(net, imagesTrain, boxesTrain, _, 0, options.OverlapsIgnore))
Console.WriteLine($"training upsampled results: {matrix}");
Console.WriteLine("num testing images: {images_test.Count()}");
using (var _ = new TestBoxOverlap())
using (var matrix = Dlib.TestObjectDetectionFunction(net, imagesTest, boxesTest, _, 0, options.OverlapsIgnore))
Console.WriteLine($"testing results: {matrix}");
Dlib.UpsampleImageDataset(2, imagesTest, boxesTest, 1800 * 1800);
using (var _ = new TestBoxOverlap())
using (var matrix = Dlib.TestObjectDetectionFunction(net, imagesTest, boxesTest, _, 0, options.OverlapsIgnore))
Console.WriteLine($"testing upsampled results: {matrix}");
/*
* This program takes many hours to execute on a high end GPU. It took about a day to
* train on a NVIDIA 1080ti. The resulting model file is available at
* http://dlib.net/files/mmod_rear_end_vehicle_detector.dat.bz2
* It should be noted that this file on dlib.net has a dlib::shape_predictor appended
* onto the end of it (see dnn_mmod_find_cars_ex.cpp for an example of its use). This
* explains why the model file on dlib.net is larger than the
* mmod_rear_end_vehicle_detector.dat output by this program.
*
* You can see some videos of this vehicle detector running on YouTube:
* https://www.youtube.com/watch?v=4B3bzmxMAZU
* https://www.youtube.com/watch?v=bP2SUo5vSlc
*
* Also, the training and testing accuracies were:
* num training images: 2217
* training results: 0.990738 0.736431 0.736073
* training upsampled results: 0.986837 0.937694 0.936912
* num testing images: 135
* testing results: 0.988827 0.471372 0.470806
* testing upsampled results: 0.987879 0.651132 0.650399
*/
}
}
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
private static IList <long> AlignPoints(IList <DPoint> from,
IList <DPoint> to,
double minAngle = -90 *Math.PI / 180.0,
double maxAngle = 90 *Math.PI / 180.0,
long numAngles = 181)
{
/*!
* ensures
* - Figures out how to align the points in from with the points in to. Returns an
* assignment array A that indicates that from[i] matches with to[A[i]].
*
* We use the Hungarian algorithm with a search over reasonable angles. This method
* works because we just need to account for a translation and a mild rotation and
* nothing else. If there is any other more complex mapping then you probably don't
* have landmarks that make sense to flip.
* !*/
if (from.Count != to.Count)
{
throw new ArgumentException();
}
long[] bestAssignment = null;
var bestAssignmentCost = double.PositiveInfinity;
using (var dists = new Matrix <double>(from.Count, to.Count))
{
foreach (var angle in Dlib.Linspace(minAngle, maxAngle, (int)numAngles))
{
using (var rot = Dlib.RotationMatrix(angle))
for (int r = 0, rows = dists.Rows; r < rows; ++r)
{
using (var tmp = rot * from[r])
for (int c = 0, columns = dists.Columns; c < columns; ++c)
{
using (var tmp2 = tmp - to[c])
dists[r, c] = Dlib.LengthSquared(tmp2);
}
}
using (var tmp = dists / Dlib.Max(dists))
using (var tmp2 = long.MaxValue * tmp)
using (var tmp3 = Dlib.Round(tmp2))
using (var idists = Dlib.MatrixCast <long>(-tmp3))
{
var assignment = Dlib.MaxCostAssignment(idists).ToArray();
var cost = Dlib.AssignmentCost(dists, assignment);
if (cost < bestAssignmentCost)
{
bestAssignmentCost = cost;
bestAssignment = assignment.ToArray();
}
}
}
// Now compute the alignment error in terms of average distance moved by each part. We
// do this so we can give the user a warning if it's impossible to make a good
// alignment.
using (var rs = new RunningStats <double>())
{
var tmp = new List <DPoint>(Enumerable.Range(0, to.Count).Select(i => new DPoint()));
for (var i = 0; i < to.Count; ++i)
{
tmp[(int)bestAssignment[i]] = to[i];
}
using (var tform = Dlib.FindSimilarityTransform(from, tmp))
for (var i = 0; i < from.Count; ++i)
{
var p = tform.Operator(from[i]) - tmp[i];
rs.Add(Dlib.Length(p));
}
if (rs.Mean > 0.05)
{
Console.WriteLine("WARNING, your dataset has object part annotations and you asked imglab to ");
Console.WriteLine("flip the data. Imglab tried to adjust the part labels so that the average");
Console.WriteLine("part layout in the flipped dataset is the same as the source dataset. ");
Console.WriteLine("However, the part annotation scheme doesn't seem to be left-right symmetric.");
Console.WriteLine("You should manually review the output to make sure the part annotations are ");
Console.WriteLine("labeled as you expect.");
}
return(bestAssignment);
}
}
}
private static void FlipDataset(CommandArgument fileArgs, CommandOption jpgOption, bool basic)
{
//var flip = parser.GetOptions().FirstOrDefault(option => option.ShortName == "flip");
//var flipBasic = parser.GetOptions().FirstOrDefault(option => option.ShortName == "flip-basic");
var dataSource = fileArgs.Value;
using (var metadata = Dlib.ImageDatasetMetadata.LoadImageDatasetMetadata(dataSource))
using (var origMetadata = Dlib.ImageDatasetMetadata.LoadImageDatasetMetadata(dataSource))
{
// Set the current directory to be the one that contains the
// metadata file. We do this because the file might contain
// file paths which are relative to this folder.
var parentDir = Path.GetDirectoryName(Path.GetFullPath(dataSource));
Environment.CurrentDirectory = parentDir;
var metadataFilename = Path.Combine(parentDir, $"flipped_{Path.GetFileName(dataSource)}");
var images = metadata.Images;
for (int i = 0, iCount = images.Count; i < iCount; ++i)
{
var f = new FileInfo(images[i].FileName);
var parent = Path.GetDirectoryName(f.FullName);
var filename = Path.Combine(parent, $"flipped_{ToPngName(f.Name)}");
using (var img = Dlib.LoadImage <RgbPixel>(images[i].FileName))
using (var temp = new Array2D <RgbPixel>())
{
Dlib.FlipImageLeftRight(img, temp);
if (jpgOption.HasValue())
{
filename = ToJpgName(filename);
Dlib.SaveJpeg(temp, filename, JpegQuality);
}
else
{
Dlib.SavePng(temp, filename);
}
var boxes = images[i].Boxes;
for (int j = 0, bCount = boxes.Count; j < bCount; ++j)
{
boxes[j].Rect = Dlib.FlipRectLeftRight(boxes[j].Rect, img.Rect);
// flip all the object parts
foreach (var kvp in boxes[j].Parts.ToArray())
{
var rect = new Rectangle(kvp.Value, kvp.Value);
var flipRect = Dlib.FlipRectLeftRight(rect, img.Rect);
boxes[j].Parts[kvp.Key] = flipRect.TopLeft;
}
}
images[i].FileName = filename;
}
}
//if (flipBasic == null || !flipBasic.HasValue())
if (!basic)
{
MakePartLabelingMatchTargetDataset(origMetadata, metadata);
}
Dlib.ImageDatasetMetadata.SaveImageDatasetMetadata(metadata, metadataFilename);
}
}
private static void Main()
{
try
{
// You can get this file from http://dlib.net/files/mmod_rear_end_vehicle_detector.dat.bz2
// This network was produced by the dnn_mmod_train_find_cars_ex.cpp example program.
// As you can see, the file also includes a separately trained shape_predictor. To see
// a generic example of how to train those refer to train_shape_predictor_ex.cpp.
using (var deserialize = new ProxyDeserialize("mmod_rear_end_vehicle_detector.dat"))
using (var net = LossMmod.Deserialize(deserialize, 1))
using (var sp = ShapePredictor.Deserialize(deserialize))
using (var img = Dlib.LoadImageAsMatrix <RgbPixel>("mmod_cars_test_image.jpg"))
using (var win = new ImageWindow())
{
win.SetImage(img);
// Run the detector on the image and show us the output.
var dets = net.Operator(img).First();
foreach (var d in dets)
{
// We use a shape_predictor to refine the exact shape and location of the detection
// box. This shape_predictor is trained to simply output the 4 corner points of
// the box. So all we do is make a rectangle that tightly contains those 4 points
// and that rectangle is our refined detection position.
var fd = sp.Detect(img, d);
var rect = Rectangle.Empty;
for (var j = 0u; j < fd.Parts; ++j)
{
rect += fd.GetPart(j);
}
win.AddOverlay(rect, new RgbPixel(255, 0, 0));
}
Console.WriteLine("Hit enter to view the intermediate processing steps");
Console.ReadKey();
// Now let's look at how the detector works. The high level processing steps look like:
// 1. Create an image pyramid and pack the pyramid into one big image. We call this
// image the "tiled pyramid".
// 2. Run the tiled pyramid image through the CNN. The CNN outputs a new image where
// bright pixels in the output image indicate the presence of cars.
// 3. Find pixels in the CNN's output image with a value > 0. Those locations are your
// preliminary car detections.
// 4. Perform non-maximum suppression on the preliminary detections to produce the
// final output.
//
// We will be plotting the images from steps 1 and 2 so you can visualize what's
// happening. For the CNN's output image, we will use the jet colormap so that "bright"
// outputs, i.e. pixels with big values, appear in red and "dim" outputs appear as a
// cold blue color. To do this we pick a range of CNN output values for the color
// mapping. The specific values don't matter. They are just selected to give a nice
// looking output image.
const float lower = -2.5f;
const float upper = 0.0f;
Console.WriteLine($"jet color mapping range: lower={lower} upper={upper}");
// Create a tiled pyramid image and display it on the screen.
// Get the type of pyramid the CNN used
//using pyramid_type = std::remove_reference < decltype(input_layer(net)) >::type::pyramid_type;
// And tell create_tiled_pyramid to create the pyramid using that pyramid type.
using (var inputLayer = new InputRgbImagePyramid <PyramidDown>(6))
{
net.TryGetInputLayer(inputLayer);
var padding = inputLayer.GetPyramidPadding();
var outerPadding = inputLayer.GetPyramidOuterPadding();
Dlib.CreateTiledPyramid <RgbPixel, PyramidDown>(img,
padding,
outerPadding,
6,
out var tiledImg,
out var rects);
using (var winpyr = new ImageWindow(tiledImg, "Tiled pyramid"))
{
// This CNN detector represents a sliding window detector with 3 sliding windows. Each
// of the 3 windows has a different aspect ratio, allowing it to find vehicles which
// are either tall and skinny, squarish, or short and wide. The aspect ratio of a
// detection is determined by which channel in the output image triggers the detection.
// Here we are just going to max pool the channels together to get one final image for
// our display. In this image, a pixel will be bright if any of the sliding window
// detectors thinks there is a car at that location.
using (var subnet = net.GetSubnet())
{
var output = subnet.Output;
Console.WriteLine($"Number of channels in final tensor image: {output.K}");
var networkOutput = Dlib.ImagePlane(output);
for (var k = 1; k < output.K; k++)
{
using (var tmpNetworkOutput = Dlib.ImagePlane(output, 0, k))
{
var maxPointWise = Dlib.MaxPointWise(networkOutput, tmpNetworkOutput);
networkOutput.Dispose();
networkOutput = maxPointWise;
}
}
// We will also upsample the CNN's output image. The CNN we defined has an 8x
// downsampling layer at the beginning. In the code below we are going to overlay this
// CNN output image on top of the raw input image. To make that look nice it helps to
// upsample the CNN output image back to the same resolution as the input image, which
// we do here.
var networkOutputScale = img.Columns / (double)networkOutput.Columns;
Dlib.ResizeImage(networkOutput, networkOutputScale);
// Display the network's output as a color image.
using (var jet = Dlib.Jet(networkOutput, upper, lower))
using (var winOutput = new ImageWindow(jet, "Output tensor from the network"))
{
// Also, overlay network_output on top of the tiled image pyramid and display it.
for (var r = 0; r < tiledImg.Rows; ++r)
{
for (var c = 0; c < tiledImg.Columns; ++c)
{
var tmp = new DPoint(c, r);
tmp = Dlib.InputTensorToOutputTensor(net, tmp);
var dp = networkOutputScale * tmp;
tmp = new DPoint((int)dp.X, (int)dp.Y);
if (Dlib.GetRect(networkOutput).Contains((int)tmp.X, (int)tmp.Y))
{
var val = networkOutput[(int)tmp.Y, (int)tmp.X];
// alpha blend the network output pixel with the RGB image to make our
// overlay.
var p = new RgbAlphaPixel();
Dlib.AssignPixel(ref p, Dlib.ColormapJet(val, lower, upper));
p.Alpha = 120;
var rgb = new RgbPixel();
Dlib.AssignPixel(ref rgb, p);
tiledImg[r, c] = rgb;
}
}
}
// If you look at this image you can see that the vehicles have bright red blobs on
// them. That's the CNN saying "there is a car here!". You will also notice there is
// a certain scale at which it finds cars. They have to be not too big or too small,
// which is why we have an image pyramid. The pyramid allows us to find cars of all
// scales.
using (var winPyrOverlay = new ImageWindow(tiledImg, "Detection scores on image pyramid"))
{
// Finally, we can collapse the pyramid back into the original image. The CNN doesn't
// actually do this step, since it's enough to threshold the tiled pyramid image to get
// the detections. However, it makes a nice visualization and clearly indicates that
// the detector is firing for all the cars.
using (var collapsed = new Matrix <float>(img.Rows, img.Columns))
using (var inputTensor = new ResizableTensor())
{
inputLayer.ToTensor(img, 1, inputTensor);
for (var r = 0; r < collapsed.Rows; ++r)
{
for (var c = 0; c < collapsed.Columns; ++c)
{
// Loop over a bunch of scale values and look up what part of network_output
// corresponds to the point(c,r) in the original image, then take the max
// detection score over all the scales and save it at pixel point(c,r).
var maxScore = -1e30f;
for (double scale = 1; scale > 0.2; scale *= 5.0 / 6.0)
{
// Map from input image coordinates to tiled pyramid coordinates.
var tensorSpace = inputLayer.ImageSpaceToTensorSpace(inputTensor, scale, new DRectangle(new DPoint(c, r)));
var tmp = tensorSpace.Center;
// Now map from pyramid coordinates to network_output coordinates.
var dp = networkOutputScale * Dlib.InputTensorToOutputTensor(net, tmp);
tmp = new DPoint((int)dp.X, (int)dp.Y);
if (Dlib.GetRect(networkOutput).Contains((int)tmp.X, (int)tmp.Y))
{
var val = networkOutput[(int)tmp.Y, (int)tmp.X];
if (val > maxScore)
{
maxScore = val;
}
}
}
collapsed[r, c] = maxScore;
// Also blend the scores into the original input image so we can view it as
// an overlay on the cars.
var p = new RgbAlphaPixel();
Dlib.AssignPixel(ref p, Dlib.ColormapJet(maxScore, lower, upper));
p.Alpha = 120;
var rgb = new RgbPixel();
Dlib.AssignPixel(ref rgb, p);
img[r, c] = rgb;
}
}
using (var jet2 = Dlib.Jet(collapsed, upper, lower))
using (var winCollapsed = new ImageWindow(jet2, "Collapsed output tensor from the network"))
using (var winImgAndSal = new ImageWindow(img, "Collapsed detection scores on raw image"))
{
Console.WriteLine("Hit enter to end program");
Console.ReadKey();
}
}
}
}
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
private static void Main()
{
try
{
var cap = new VideoCapture(0);
//var cap = new VideoCapture("20090124_WeeklyAddress.ogv.360p.webm");
if (!cap.IsOpened())
{
Console.WriteLine("Unable to connect to camera");
return;
}
using (var win = new ImageWindow())
{
// Load face detection and pose estimation models.
using (var detector = Dlib.GetFrontalFaceDetector())
using (var poseModel = ShapePredictor.Deserialize("shape_predictor_68_face_landmarks.dat"))
{
// Grab and process frames until the main window is closed by the user.
while (!win.IsClosed())
{
// Grab a frame
var temp = new Mat();
if (!cap.Read(temp))
{
break;
}
// Turn OpenCV's Mat into something dlib can deal with. Note that this just
// wraps the Mat object, it doesn't copy anything. So cimg is only valid as
// long as temp is valid. Also don't do anything to temp that would cause it
// to reallocate the memory which stores the image as that will make cimg
// contain dangling pointers. This basically means you shouldn't modify temp
// while using cimg.
var array = new byte[temp.Width * temp.Height * temp.ElemSize()];
Marshal.Copy(temp.Data, array, 0, array.Length);
using (var cimg = Dlib.LoadImageData <RgbPixel>(array, (uint)temp.Height, (uint)temp.Width, (uint)(temp.Width * temp.ElemSize())))
{
// Detect faces
var faces = detector.Detect(cimg);
// Find the pose of each face.
var shapes = new List <FullObjectDetection>();
for (var i = 0; i < faces.Length; ++i)
{
var det = poseModel.Detect(cimg, faces[i]);
shapes.Add(det);
}
// Display it all on the screen
win.ClearOverlay();
win.SetImage(cimg);
var lines = Dlib.RenderFaceDetections(shapes);
win.AddOverlay(lines);
foreach (var line in lines)
{
line.Dispose();
}
}
}
}
}
}
//catch (serialization_error&e)
//{
// cout << "You need dlib's default face landmarking model file to run this example." << endl;
// cout << "You can get it from the following URL: " << endl;
// cout << " http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2" << endl;
// cout << endl << e.what() << endl;
//}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
private static void Main(string[] args)
{
try
{
if (args.Length == 0)
{
Console.WriteLine("Give an image dataset XML file to run this program.");
Console.WriteLine("For example, if you are running from the examples folder then run this program by typing");
Console.WriteLine(" ./RandomCropper faces/training.xml");
return;
}
// First lets load a dataset
IList <Matrix <RgbPixel> > images;
IList <IList <MModRect> > boxes;
Dlib.LoadImageDataset(args[0], out images, out boxes);
// Here we make our random_cropper. It has a number of options.
var cropper = new DlibDotNet.ImageTransforms.RandomCropper();
// We can tell it how big we want the cropped images to be.
cropper.ChipDims = new ChipDims(400, 400);
// Also, when doing cropping, it will map the object annotations from the
// dataset to the cropped image as well as perform random scale jittering.
// You can tell it how much scale jittering you would like by saying "please
// make the objects in the crops have a min and max size of such and such".
// You do that by calling these two functions. Here we are saying we want the
// objects in our crops to be no more than 0.8*400 pixels in height and width.
cropper.MaxObjectSize = 0.8;
// And also that they shouldn't be too small. Specifically, each object's smallest
// dimension (i.e. height or width) should be at least 60 pixels and at least one of
// the dimensions must be at least 80 pixels. So the smallest objects the cropper will
// output will be either 80x60 or 60x80.
cropper.MinObjectLengthLongDim = 80;
cropper.MinObjectLengthShortDim = 60;
// The cropper can also randomly mirror and rotate crops, which we ask it to
// perform as well.
cropper.RandomlyFlip = true;
cropper.MaxRotationDegrees = 50;
// This fraction of crops are from random parts of images, rather than being centered
// on some object.
cropper.BackgroundCropsFraction = 0.2;
// Now ask the cropper to generate a bunch of crops. The output is stored in
// crops and crop_boxes.
IEnumerable <Matrix <RgbPixel> > crops;
IEnumerable <IEnumerable <MModRect> > cropBoxes;
// Make 1000 crops.
cropper.Operator(1000, images, boxes, out crops, out cropBoxes);
// Finally, lets look at the results
var cropList = crops?.ToArray() ?? new Matrix <RgbPixel> [0];
var cropBoxesList = cropBoxes?.ToArray() ?? new IEnumerable <MModRect> [0];
using (var win = new ImageWindow())
for (var i = 0; i < cropList.Count(); ++i)
{
win.ClearOverlay();
win.SetImage(cropList[i]);
foreach (var b in cropBoxesList[i])
{
// Note that mmod_rect has an ignore field. If an object was labeled
// ignore in boxes then it will still be labeled as ignore in
// crop_boxes. Moreover, objects that are not well contained within
// the crop are also set to ignore.
var rect = b.Rect;
if (b.Ignore)
{
win.AddOverlay(rect, new RgbPixel {
Red = 255, Blue = 255
}); // draw ignored boxes as orange
}
else
{
win.AddOverlay(rect, new RgbPixel {
Red = 255
}); // draw other boxes as red
}
}
Console.WriteLine("Hit enter to view the next random crop.");
Console.ReadKey();
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public void ExtracFHogFeatures()
{
const string testName = "ExtracFHogFeatures";
var path = this.GetDataFile($"{LoadTarget}.bmp");
var tests = new[]
{
new { Type = MatrixElementTypes.Float, ExpectResult = true },
new { Type = MatrixElementTypes.Double, ExpectResult = true },
new { Type = MatrixElementTypes.RgbPixel, ExpectResult = false },
new { Type = MatrixElementTypes.RgbAlphaPixel, ExpectResult = false },
new { Type = MatrixElementTypes.HsiPixel, ExpectResult = false },
new { Type = MatrixElementTypes.UInt32, ExpectResult = false },
new { Type = MatrixElementTypes.UInt8, ExpectResult = false },
new { Type = MatrixElementTypes.UInt16, ExpectResult = false },
new { Type = MatrixElementTypes.Int8, ExpectResult = false },
new { Type = MatrixElementTypes.Int16, ExpectResult = false },
new { Type = MatrixElementTypes.Int32, ExpectResult = false }
};
foreach (ImageTypes inputType in Enum.GetValues(typeof(ImageTypes)))
{
foreach (var output in tests)
{
if (inputType == ImageTypes.Matrix)
{
continue;
}
var expectResult = output.ExpectResult;
var imageObj = DlibTest.LoadImage(inputType, path);
Array2DMatrixBase outputObj = null;
var outputImageAction = new Func <bool, Array2DMatrixBase>(expect =>
{
switch (output.Type)
{
case MatrixElementTypes.UInt8:
outputObj = Dlib.ExtracFHogFeatures <byte>(imageObj);
break;
case MatrixElementTypes.UInt16:
outputObj = Dlib.ExtracFHogFeatures <ushort>(imageObj);
break;
case MatrixElementTypes.UInt32:
outputObj = Dlib.ExtracFHogFeatures <uint>(imageObj);
break;
case MatrixElementTypes.Int8:
outputObj = Dlib.ExtracFHogFeatures <sbyte>(imageObj);
break;
case MatrixElementTypes.Int16:
outputObj = Dlib.ExtracFHogFeatures <short>(imageObj);
break;
case MatrixElementTypes.Int32:
outputObj = Dlib.ExtracFHogFeatures <int>(imageObj);
break;
case MatrixElementTypes.Float:
outputObj = Dlib.ExtracFHogFeatures <float>(imageObj);
break;
case MatrixElementTypes.Double:
outputObj = Dlib.ExtracFHogFeatures <double>(imageObj);
break;
case MatrixElementTypes.RgbPixel:
outputObj = Dlib.ExtracFHogFeatures <RgbPixel>(imageObj);
break;
case MatrixElementTypes.RgbAlphaPixel:
outputObj = Dlib.ExtracFHogFeatures <RgbAlphaPixel>(imageObj);
break;
case MatrixElementTypes.HsiPixel:
outputObj = Dlib.ExtracFHogFeatures <HsiPixel>(imageObj);
break;
default:
throw new ArgumentOutOfRangeException();
}
return(outputObj);
});
var successAction = new Action <Array2DMatrixBase>(image =>
{
MatrixBase ret = null;
try
{
ret = Dlib.DrawHog(image);
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
finally
{
if (ret != null)
{
this.DisposeAndCheckDisposedState(ret);
}
}
});
var failAction = new Action(() =>
{
Assert.Fail($"{testName} should throw excption for InputType: {inputType}, OutputType: {output.Type}.");
});
var finallyAction = new Action(() =>
{
if (imageObj != null)
{
this.DisposeAndCheckDisposedState(imageObj);
}
if (outputObj != null)
{
this.DisposeAndCheckDisposedState(outputObj);
}
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {inputType}, OutputType: {output.Type}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
}
private void DataButton2_Click(object sender, EventArgs e)
{
openFileDialog1.Filter = @"文本文件(*.txt)|*.txt";
openFileDialog1.RestoreDirectory = true;
bool isLoad = false;
switch (DataLoadModeGuide)
{
case 0:
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
string path = openFileDialog1.FileName;
StreamReader sr = new StreamReader(path, Encoding.Default);
String[] data = { };
char[] separator = { '\n', '\r', ' ' };
data = sr.ReadToEnd().Split(separator);
for (int i = 0; i < 68 * 2; i++)
{
PointFaceGuide[i] = (int)Convert.ToSingle(data[i]);
}
isLoad = true;
}
break;
case 1:
var detector = Dlib.GetFrontalFaceDetector();
var img = Dlib.LoadImage <RgbPixel>(GuidePath);
Dlib.PyramidUp(img);
var dets = detector.Operator(img);
//var temp = dets.Length;
var shapes = new List <FullObjectDetection>();
var sp = ShapePredictor.Deserialize("./model.dat");
foreach (var rect in dets)
{
var shape = sp.Detect(img, rect);
for (int i = 0; i < 68; i++)
{
PointFaceGuide[i * 2] = shape.GetPart((uint)i).X / 2;
PointFaceGuide[i * 2 + 1] = shape.GetPart((uint)i).Y / 2;
}
if (shape.Parts > 2)
{
shapes.Add(shape);
}
}
isLoad = true;
break;
default:
throw new IndexOutOfRangeException();
}
if (isLoad)
{
//显示标记点
var g = Graphics.FromImage(ImgGuideShow);
for (int i = 0; i < 68; i++)
{
g.FillEllipse(new SolidBrush(Color.FromArgb(255, 255, 220)), (int)PointFaceGuide[i * 2], (int)PointFaceGuide[i * 2 + 1], 4, 4);
}
g.Dispose();
pictureBox2.Image = ImgGuideShow.Clone() as Image;
if (DataButton.Enabled == true && DataButton2.Enabled == true)
{
RunButton.Enabled = true;
}
}
}
public void TryTrack()
{
const string testName = "TryTrack";
var tests = new[]
{
new { Type = ImageTypes.Int32, ExpectResult = true }
};
var type = this.GetType().Name;
foreach (var input in tests)
{
var tracker = new CorrelationTracker();
try
{
var index = 0;
foreach (var file in this.GetDataFiles("video_frames").Where(info => info.Name.EndsWith("jpg")))
{
var expectResult = input.ExpectResult;
var inputType = input.Type;
var imageObj = DlibTest.LoadImage(input.Type, file);
if (index == 0)
{
var rect = DRectangle.CenteredRect(93, 110, 38, 86);
tracker.StartTrack(imageObj, rect);
}
var outputImageAction = new Func <bool, Array2DBase>(expect =>
{
if (index != 0)
{
tracker.Update(imageObj);
}
return(imageObj);
});
var successAction = new Action <Array2DBase>(image =>
{
if (index != 0)
{
tracker.Update(image);
}
var r = tracker.GetPosition();
using (var img = Dlib.LoadImage <RgbPixel>(file.FullName))
{
Dlib.DrawRectangle(img, (Rectangle)r, new RgbPixel {
Red = 255
}, 3);
Dlib.SaveJpeg(img, Path.Combine(this.GetOutDir(type), $"{Path.GetFileNameWithoutExtension(file.FullName)}.jpg"));
}
});
var failAction = new Action(() =>
{
Assert.Fail($"{testName} should throw excption for InputType: {inputType}.");
});
var finallyAction = new Action(() =>
{
index++;
if (imageObj != null)
{
this.DisposeAndCheckDisposedState(imageObj);
}
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {inputType}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
finally
{
this.DisposeAndCheckDisposedState(tracker);
}
}
}
public void GetFrontalFaceDetector()
{
this._FrontalFaceDetector = Dlib.GetFrontalFaceDetector();
}
private void BackgroundWorkerOnDoWork(object sender, DoWorkEventArgs doWorkEventArgs)
{
var path = doWorkEventArgs.Argument as string;
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
{
return;
}
using (var faceDetector = Dlib.GetFrontalFaceDetector())
using (var img = Dlib.LoadImage <RgbPixel>(path))
{
Dlib.PyramidUp(img);
var dets = faceDetector.Operator(img);
var shapes = new List <FullObjectDetection>();
foreach (var rect in dets)
{
var shape = this._ShapePredictor.Detect(img, rect);
if (shape.Parts <= 2)
{
continue;
}
shapes.Add(shape);
}
if (shapes.Any())
{
var lines = Dlib.RenderFaceDetections(shapes);
foreach (var line in lines)
{
Dlib.DrawLine(img, line.Point1, line.Point2, new RgbPixel
{
Green = 255
});
}
var wb = img.ToBitmap();
this.pictureBoxImage.Image?.Dispose();
this.pictureBoxImage.Image = wb;
foreach (var l in lines)
{
l.Dispose();
}
var chipLocations = Dlib.GetFaceChipDetails(shapes);
using (var faceChips = Dlib.ExtractImageChips <RgbPixel>(img, chipLocations))
using (var tileImage = Dlib.TileImages(faceChips))
{
// It is NOT necessary to re-convert WriteableBitmap to Matrix.
// This sample demonstrate converting managed image class to
// dlib class and vice versa.
using (var tile = tileImage.ToBitmap())
using (var mat = tile.ToMatrix <RgbPixel>())
{
var tile2 = mat.ToBitmap();
this.pictureBoxTileImage.Image?.Dispose();
this.pictureBoxTileImage.Image = tile2;
}
}
foreach (var c in chipLocations)
{
c.Dispose();
}
}
foreach (var s in shapes)
{
s.Dispose();
}
}
}
public async Task ProcessAsync(string[] inputFilenames)
{
var chips = new List <Matrix <RgbPixel> >();
var faces = new List <Rectangle>();
var filename = new List <string>();
var jsonFilename = inputFilenames.First() + ".json";
foreach (var inputFilename in inputFilenames)
{
if (!File.Exists(inputFilename))
{
break;
}
if (File.Exists(jsonFilename))
{
continue;
}
// load the image
using var img = await DlibHelpers.LoadRotatedImage(imageRotationService, inputFilename);
// Dlib.SaveJpeg(img, inputFilename + "__1.jpg", 25);
// Dlib.SaveJpeg(img, inputFilename + "__2.jpg", 25);
// detect all faces
foreach (var face in detector.Operator(img))
{
// detect landmarks
var shape = predictor.Detect(img, face);
// extract normalized and rotated 150x150 face chip
var faceChipDetail = Dlib.GetFaceChipDetails(shape, 150, 0.25);
var faceChip = Dlib.ExtractImageChip <RgbPixel>(img, faceChipDetail);
// convert the chip to a matrix and store
var matrix = new Matrix <RgbPixel>(faceChip);
chips.Add(matrix);
faces.Add(face);
filename.Add(inputFilename);
}
}
if (!File.Exists(jsonFilename))
{
var ffd = new FoundFacesData
{
// Chips = chips,
Faces = faces,
Filenames = filename,
};
OutputLabels <Matrix <float> > descriptors = null;
if (chips.Any())
{
// put each fae in a 128D embedding space
// similar faces will be placed close together
// Console.WriteLine("Recognizing faces...");
descriptors = dnn.Operator(chips);
ffd.Descriptors = descriptors.ToList();
}
else
{
ffd.Descriptors = new List <Matrix <float> >(0);
}
var dto = new FoundFacesDataDto
{
Faces = ffd.Faces
.Select(f => new RectangleDto
{
Bottom = f.Bottom,
Left = f.Left,
Top = f.Top,
Right = f.Right,
})
.ToList(),
Filenames = ffd.Filenames,
Descriptors = ffd.Descriptors
.Select(x => new MatrixFloatDto
{
Data = x.ToArray(),
Row = x.Rows,
Columns = x.Columns,
})
.ToList()
};
var x = JsonConvert.SerializeObject(dto);
File.WriteAllText(jsonFilename, JsonConvert.SerializeObject(dto));
}
FoundFacesData items;
using (var r = new StreamReader(jsonFilename))
{
var json = r.ReadToEnd();
var itemsdto = JsonConvert.DeserializeObject <FoundFacesDataDto>(json);
items = new FoundFacesData
{
Faces = itemsdto.Faces.Select(f => new Rectangle(f.Left, f.Top, f.Right, f.Bottom)).ToList(),
Filenames = itemsdto.Filenames.ToList(),
Descriptors = itemsdto.Descriptors.Select(d => new Matrix <float>(d.Data, d.Row, d.Columns)).ToList(),
};
}
if (items.Faces.Count <= 0)
{
return;
}
// // compare each face with all other faces
var edges = new List <SamplePair>();
// for (uint i = 0; i < descriptors.Count; ++i)
// for (var j = i; j < descriptors.Count; ++j)
//
// // record every pair of two similar faces
// // faces are similar if they are less than 0.6 apart in the 128D embedding space
// if (Dlib.Length(descriptors[i] - descriptors[j]) < 0.5)
// edges.Add(new SamplePair(i, j));
//
// // use the chinese whispers algorithm to find all face clusters
// Dlib.ChineseWhispers(edges, 100, out var clusters, out var labels);
// // Console.WriteLine($" Found {clusters} unique person(s) in the image");
//
// // draw rectangles on each face using the cluster color
// for (var i = 0; i < faces.Count; i++)
// {
// var color = new RgbPixel(255, 255, 255);
// if (labels[i] < palette.Length)
// color = palette[labels[i]];
//
// using var img = Dlib.LoadImage<RgbPixel>(filename[i] + "__1.jpg");
// Dlib.DrawRectangle(img, faces[i], color: color, thickness: 4);
// Dlib.SaveJpeg(img, filename[i] + "__1.jpg", 25);
// }
//
// Console.WriteLine("end 1");
// compare each face with all other faces
edges = new List <SamplePair>();
for (var i = 0; i < items.Descriptors.Count; ++i)
{
for (var j = i; j < items.Descriptors.Count; ++j)
{
// record every pair of two similar faces
// faces are similar if they are less than 0.6 apart in the 128D embedding space
if (Dlib.Length(items.Descriptors[i] - items.Descriptors[j]) < 0.4)
{
edges.Add(new SamplePair((uint)i, (uint)j));
}
}
}
// use the chinese whispers algorithm to find all face clusters
Dlib.ChineseWhispers(edges, 100, out var clusters2, out var labels2);
// Console.WriteLine($" Found {clusters} unique person(s) in the image");
// draw rectangles on each face using the cluster color
for (var i = 0; i < items.Faces.Count; i++)
{
var color = palette[0];
if (labels2[i] < palette.Length)
{
color = palette[labels2[i]];
}
if (!File.Exists(items.Filenames[i] + $"_x{labels2[i]}.jpg"))
{
using var img2 = await DlibHelpers.LoadRotatedImage(imageRotationService, items.Filenames[i]);
Dlib.SaveJpeg(img2, items.Filenames[i] + $"_x{labels2[i]}.jpg", 25);
}
using var img = Dlib.LoadImage <RgbPixel>(items.Filenames[i] + $"_x{labels2[i]}.jpg");
Dlib.DrawRectangle(img, items.Faces[i], color: color, thickness: 4);
Dlib.SaveJpeg(img, items.Filenames[i] + $"_x{labels2[i]}.jpg", 25);
}
// var origFilename = new FileInfo(inputFilename).Name;
// var outputFilename = Path.Combine(outputDirectory, $"{origFilename}_Identification.jpg");
// Dlib.SaveJpeg(img, inputFilename, 75);
}
public void FlipImageLeftRight2()
{
const string testName = "FlipImageLeftRight2";
var path = this.GetDataFile($"{LoadTarget}.bmp");
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true }
};
var type = this.GetType().Name;
foreach (var input in tests)
{
foreach (var output in tests)
{
var expectResult = input.ExpectResult;
var imageObj = DlibTest.LoadImage(input.Type, path);
var outputObj = Array2DTest.CreateArray2D(output.Type);
var outputImageAction = new Func <bool, Array2DBase>(expect =>
{
Dlib.FlipImageLeftRight(imageObj, outputObj);
return(outputObj);
});
var successAction = new Action <Array2DBase>(image =>
{
Dlib.SaveBmp(outputObj, $"{Path.Combine(this.GetOutDir(type, testName), $"{LoadTarget}_{input.Type}_{output.Type}.bmp")}");
});
var failAction = new Action(() =>
{
Assert.Fail($"{testName} should throw exception for InputType: {input.Type}, OutputType: {output.Type}.");
});
var finallyAction = new Action(() =>
{
if (imageObj != null)
{
this.DisposeAndCheckDisposedState(imageObj);
}
if (outputObj != null)
{
this.DisposeAndCheckDisposedState(outputObj);
}
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {input.Type}, OutputType: {output.Type}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
}
private static void Main()
{
try
{
// The API for doing metric learning is very similar to the API for
// multi-class classification. In fact, the inputs are the same, a bunch of
// labeled objects. So here we create our dataset. We make up some simple
// vectors and label them with the integers 1,2,3,4. The specific values of
// the integer labels don't matter.
var samples = new List <Matrix <double> >();
var labels = new List <uint>();
// class 1 training vectors
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 1, 0, 0, 0, 0, 0, 0, 0 })); labels.Add(1);
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 0, 1, 0, 0, 0, 0, 0, 0 })); labels.Add(1);
// class 2 training vectors
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 0, 0, 1, 0, 0, 0, 0, 0 })); labels.Add(2);
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 0, 0, 0, 1, 0, 0, 0, 0 })); labels.Add(2);
// class 3 training vectors
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 0, 0, 0, 0, 1, 0, 0, 0 })); labels.Add(3);
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 0, 0, 0, 0, 0, 1, 0, 0 })); labels.Add(3);
// class 4 training vectors
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 0, 0, 0, 0, 0, 0, 1, 0 })); labels.Add(4);
samples.Add(new Matrix <double>(new MatrixTemplateSizeParameter(0, 1), new double[] { 0, 0, 0, 0, 0, 0, 0, 1 })); labels.Add(4);
// Make a network that simply learns a linear mapping from 8D vectors to 2D
// vectors.
using (var net = new LossMetric(1))
using (var trainer = new DnnTrainer <LossMetric>(net))
{
trainer.SetLearningRate(0.1);
// It should be emphasized out that it's really important that each mini-batch contain
// multiple instances of each class of object. This is because the metric learning
// algorithm needs to consider pairs of objects that should be close as well as pairs
// of objects that should be far apart during each training step. Here we just keep
// training on the same small batch so this constraint is trivially satisfied.
while (trainer.GetLearningRate() >= 1e-4)
{
LossMetric.TrainOneStep(trainer, samples, labels);
}
// Wait for training threads to stop
trainer.GetNet().Dispose();
Console.WriteLine("done training");
// Run all the samples through the network to get their 2D vector embeddings.
var embedded = net.Operator(samples);
// Print the embedding for each sample to the screen. If you look at the
// outputs carefully you should notice that they are grouped together in 2D
// space according to their label.
for (var i = 0; i < embedded.Count(); ++i)
{
using (var trans = Dlib.Trans(embedded[i]))
Console.Write($"label: {labels[i]}\t{trans}");
}
// Now, check if the embedding puts things with the same labels near each other and
// things with different labels far apart.
var numRight = 0;
var numWrong = 0;
for (var i = 0; i < embedded.Count(); ++i)
{
for (var j = i + 1; j < embedded.Count(); ++j)
{
if (labels[i] == labels[j])
{
// The loss_metric layer will cause things with the same label to be less
// than net.loss_details().get_distance_threshold() distance from each
// other. So we can use that distance value as our testing threshold for
// "being near to each other".
if (Dlib.Length(embedded[i] - embedded[j]) < net.GetLossDetails().GetDistanceThreshold())
{
++numRight;
}
else
{
++numWrong;
}
}
else
{
if (Dlib.Length(embedded[i] - embedded[j]) >= net.GetLossDetails().GetDistanceThreshold())
{
++numRight;
}
else
{
++numWrong;
}
}
}
}
Console.WriteLine($"num_right: {numRight}");
Console.WriteLine($"num_wrong: {numWrong}");
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public void Jet2()
{
var path = this.GetDataFile($"{LoadTarget}.bmp");
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = false, Max = 255, Min = 0 },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = false, Max = 255, Min = 0 },
new { Type = ImageTypes.UInt8, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.UInt16, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.UInt32, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.Int8, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.Int16, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.Int32, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.HsiPixel, ExpectResult = false, Max = 255, Min = 0 },
new { Type = ImageTypes.Float, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.Double, ExpectResult = true, Max = 255, Min = 0 },
new { Type = ImageTypes.RgbPixel, ExpectResult = false, Max = 75, Min = 50 },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = false, Max = 75, Min = 50 },
new { Type = ImageTypes.UInt8, ExpectResult = true, Max = 75, Min = 50 },
new { Type = ImageTypes.UInt16, ExpectResult = true, Max = 75, Min = 50 },
new { Type = ImageTypes.UInt32, ExpectResult = true, Max = 75, Min = 50 },
new { Type = ImageTypes.Int8, ExpectResult = true, Max = 75, Min = 50 },
new { Type = ImageTypes.Int16, ExpectResult = true, Max = 75, Min = 50 },
new { Type = ImageTypes.Int32, ExpectResult = true, Max = 75, Min = 50 },
new { Type = ImageTypes.HsiPixel, ExpectResult = false, Max = 75, Min = 50 },
new { Type = ImageTypes.Float, ExpectResult = true, Max = 75, Min = 50 },
new { Type = ImageTypes.Double, ExpectResult = true, Max = 75, Min = 50 }
};
var type = this.GetType().Name;
foreach (var test in tests)
{
Array2DBase imageObj = null;
Array2DBase horzObj = null;
Array2DBase vertObj = null;
Array2DBase outputImageObj = null;
MatrixOp matrix = null;
DlibObject windowObj = null;
try
{
const ImageTypes inputType = ImageTypes.Float;
var image = DlibTest.LoadImage(test.Type, path);
imageObj = image;
var horz = Array2DTest.CreateArray2D(inputType);
horzObj = horz;
var vert = Array2DTest.CreateArray2D(inputType);
vertObj = vert;
var outputImage = Array2DTest.CreateArray2D(test.Type);
outputImageObj = outputImage;
Dlib.SobelEdgeDetector(image, horz, vert);
Dlib.SuppressNonMaximumEdges(horz, vert, outputImage);
try
{
matrix = Jet(test.Type, outputImage, test.Max, test.Min);
if (test.ExpectResult)
{
if (this.CanGuiDebug)
{
var window = new ImageWindow(matrix, $"{test.Type} - Max: {test.Max}, Min : {test.Min}");
windowObj = window;
}
Dlib.SaveBmp(image, $"{Path.Combine(this.GetOutDir(type, "Jet2"), $"{LoadTarget}_{test.Type}_{test.Max}_{test.Min}.bmp")}");
}
else
{
Assert.Fail($"Failed to execute Jet2 to Type: {test.Type}");
}
}
catch (Exception)
{
if (!test.ExpectResult)
{
Console.WriteLine("OK");
}
else
{
throw;
}
}
}
catch (Exception e)
{
Console.WriteLine(e.StackTrace);
Console.WriteLine($"Failed to execute Jet2 to Type: {test.Type}");
throw;
}
finally
{
if (outputImageObj != null)
{
this.DisposeAndCheckDisposedState(outputImageObj);
}
if (vertObj != null)
{
this.DisposeAndCheckDisposedState(vertObj);
}
if (horzObj != null)
{
this.DisposeAndCheckDisposedState(horzObj);
}
if (windowObj != null)
{
this.DisposeAndCheckDisposedState(windowObj);
}
if (matrix != null)
{
this.DisposeAndCheckDisposedState(matrix);
}
if (imageObj != null)
{
this.DisposeAndCheckDisposedState(imageObj);
}
}
}
}
private static void Main()
{
try
{
// 定义图像捕捉方式 从摄像头 ， 注意 Windows下需要选择 VideoCaptureAPIs.DSHOW
var cap = new VideoCapture(0, VideoCaptureAPIs.DSHOW);
// 定义图像捕捉方式 从摄像头 视频文件
//var cap = new VideoCapture("video.webm");
//判断捕捉设备是否打开
if (!cap.IsOpened())
{
Console.WriteLine("Unable to connect to camera");
return;
}
Mat temp = null;
//定义显示窗口
using (var win = new ImageWindow())
{
//读取人脸检测和标注模型
using (var detector = Dlib.GetFrontalFaceDetector())
using (var poseModel = ShapePredictor.Deserialize("shape_predictor_68_face_landmarks.dat"))
{
// 主窗口是否关闭
while (!win.IsClosed())
{
//System.Threading.Thread.Sleep(100);
//获得1帧图片
temp = cap.RetrieveMat();// new Mat();
if (temp == null)
{
break;
}
//将 OPENCV 图像数据 转换为 DILB 图像格式
var array = new byte[temp.Width * temp.Height * temp.ElemSize()];
Marshal.Copy(temp.Data, array, 0, array.Length);
using (var cimg = Dlib.LoadImageData <BgrPixel>(array, (uint)temp.Height, (uint)temp.Width, (uint)(temp.Width * temp.ElemSize())))
{
// 人脸检测
var faces = detector.Operator(cimg);
//标注人脸
var shapes = new List <FullObjectDetection>();
for (var i = 0; i < faces.Length; ++i)
{
var det = poseModel.Detect(cimg, faces[i]);
shapes.Add(det);
}
//显示
win.ClearOverlay();
win.SetImage(cimg);
var lines = Dlib.RenderFaceDetections(shapes);
win.AddOverlay(lines);
foreach (var line in lines)
{
line.Dispose();
}
}
}
}
}
}
//catch (serialization_error&e)
//{
// cout << "需要下载识别模型 http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2" << endl;
// cout << endl << e.what() << endl;
//}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
private static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("You call this program like this: ");
Console.WriteLine("./dnn_semantic_segmentation_train_ex /path/to/images");
Console.WriteLine();
Console.WriteLine($"You will also need a trained '{SemanticSegmentationNetFilename}' file.");
Console.WriteLine("You can either train it yourself (see example program");
Console.WriteLine("dnn_semantic_segmentation_train_ex), or download a");
Console.WriteLine($"copy from here: http://dlib.net/files/{SemanticSegmentationNetFilename}");
return;
}
try
{
// Read the file containing the trained network from the working directory.
using (var net = LossMulticlassLogPerPixel.Deserialize(SemanticSegmentationNetFilename, 3))
{
// Show inference results in a window.
using (var win = new ImageWindow())
{
// Find supported image files.
var files = Directory.GetFiles(args[0])
.Where(s => s.EndsWith(".jpeg") || s.EndsWith(".jpg") || s.EndsWith(".png")).ToArray();
Console.WriteLine($"Found {files.Length} images, processing...");
foreach (var file in files)
{
// Load the input image.
using (var inputImage = Dlib.LoadImageAsMatrix <RgbPixel>(file))
{
// Create predictions for each pixel. At this point, the type of each prediction
// is an index (a value between 0 and 20). Note that the net may return an image
// that is not exactly the same size as the input.
using (var output = net.Operator(inputImage))
using (var temp = output.First())
{
// Crop the returned image to be exactly the same size as the input.
var rect = Rectangle.CenteredRect((int)(temp.Columns / 2d), (int)(temp.Rows / 2d), (uint)inputImage.Columns, (uint)inputImage.Rows);
using (var dims = new ChipDims((uint)inputImage.Rows, (uint)inputImage.Columns))
using (var chipDetails = new ChipDetails(rect, dims))
using (var indexLabelImage = Dlib.ExtractImageChip <ushort>(temp, chipDetails, InterpolationTypes.NearestNeighbor))
{
// Convert the indexes to RGB values.
using (var rgbLabelImage = IndexLabelImageToRgbLabelImage(indexLabelImage))
{
// Show the input image on the left, and the predicted RGB labels on the right.
using (var joinedRow = Dlib.JoinRows(inputImage, rgbLabelImage))
{
win.SetImage(joinedRow);
// Find the most prominent class label from amongst the per-pixel predictions.
var classLabel = GetMostProminentNonBackgroundClassLabel(indexLabelImage);
Console.WriteLine($"{file} : {classLabel} - hit enter to process the next image");
Console.ReadKey();
}
}
}
}
}
}
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public void DetectFace2()
{
if (this._ShapePredictor == null)
{
Assert.Fail("ShapePredictor is not initialized!!");
}
const string testName = "DetectFace2";
var path = this.GetDataFile("Lenna_mini.bmp");
var tests = new[]
{
new { Type = ImageTypes.RgbPixel, ExpectResult = true },
new { Type = ImageTypes.UInt8, ExpectResult = true },
new { Type = ImageTypes.UInt16, ExpectResult = true },
new { Type = ImageTypes.UInt32, ExpectResult = true },
new { Type = ImageTypes.Int8, ExpectResult = true },
new { Type = ImageTypes.Int16, ExpectResult = true },
new { Type = ImageTypes.Int32, ExpectResult = true },
new { Type = ImageTypes.HsiPixel, ExpectResult = true },
new { Type = ImageTypes.Float, ExpectResult = true },
new { Type = ImageTypes.Double, ExpectResult = true },
new { Type = ImageTypes.RgbAlphaPixel, ExpectResult = false }
};
using (var faceDetector = Dlib.GetFrontalFaceDetector())
foreach (var input in tests)
{
var expectResult = input.ExpectResult;
var imageObj = DlibTest.LoadImage(input.Type, path);
Rectangle[] dets = null;
var outputImageAction = new Func <bool, Array2DBase>(expect =>
{
dets = faceDetector.Operator(imageObj);
return(imageObj);
});
var successAction = new Action <Array2DBase>(image =>
{
var rects = new List <Rectangle>();
const int offset = 1;
var shapes = dets.Select(r => this._ShapePredictor.Detect(image, r)).ToList();
foreach (var shape in shapes)
{
var r = shape.Rect;
var parts = shape.Parts;
for (uint i = 0; i < parts; i++)
{
var part = shape.GetPart(i);
var pr = new Rectangle(
part.X - offset, part.Y - offset, part.X + offset, part.Y + offset);
rects.Add(pr);
}
rects.Add(r);
}
// This test does NOT check whether output image and detect face area are correct
//Dlib.SaveJpeg(image, $"{Path.Combine(this.GetOutDir(type, testName), $"2008_001322_{input.Type}.jpg")}");
});
var failAction = new Action(() =>
{
Assert.Fail($"{testName} should throw exception for InputType: {input.Type}.");
});
var finallyAction = new Action(() =>
{
this.DisposeAndCheckDisposedState(imageObj);
});
var exceptionAction = new Action(() =>
{
Console.WriteLine($"Failed to execute {testName} to InputType: {input.Type}.");
});
DoTest(outputImageAction, expectResult, successAction, finallyAction, failAction, exceptionAction);
}
}
private static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Run this example by invoking it like this: ");
Console.WriteLine(" ./DnnFaceRecognition faces/bald_guys.jpg");
Console.WriteLine("You will also need to get the face landmarking model file as well as ");
Console.WriteLine("the face recognition model file. Download and then decompress these files from: ");
Console.WriteLine("http://dlib.net/files/shape_predictor_5_face_landmarks.dat.bz2");
Console.WriteLine("http://dlib.net/files/dlib_face_recognition_resnet_model_v1.dat.bz2");
return;
}
// The first thing we are going to do is load all our models. First, since we need to
// find faces in the image we will need a face detector:
using (var detector = Dlib.GetFrontalFaceDetector())
// We will also use a face landmarking model to align faces to a standard pose: (see face_landmark_detection_ex.cpp for an introduction)
using (var sp = ShapePredictor.Deserialize("shape_predictor_5_face_landmarks.dat"))
// And finally we load the DNN responsible for face recognition.
using (var net = DlibDotNet.Dnn.LossMetric.Deserialize("dlib_face_recognition_resnet_model_v1.dat"))
using (var img = Dlib.LoadImageAsMatrix <RgbPixel>(args[0]))
// Display the raw image on the screen
using (var win = new ImageWindow(img))
{
// Run the face detector on the image of our action heroes, and for each face extract a
// copy that has been normalized to 150x150 pixels in size and appropriately rotated
// and centered.
var faces = new List <Matrix <RgbPixel> >();
foreach (var face in detector.Operator(img))
{
var shape = sp.Detect(img, face);
var faceChipDetail = Dlib.GetFaceChipDetails(shape, 150, 0.25);
var faceChip = Dlib.ExtractImageChip <RgbPixel>(img, faceChipDetail);
//faces.Add(move(face_chip));
faces.Add(faceChip);
// Also put some boxes on the faces so we can see that the detector is finding
// them.
win.AddOverlay(face);
}
if (!faces.Any())
{
Console.WriteLine("No faces found in image!");
return;
}
// This call asks the DNN to convert each face image in faces into a 128D vector.
// In this 128D vector space, images from the same person will be close to each other
// but vectors from different people will be far apart. So we can use these vectors to
// identify if a pair of images are from the same person or from different people.
var faceDescriptors = net.Operator(faces);
// In particular, one simple thing we can do is face clustering. This next bit of code
// creates a graph of connected faces and then uses the Chinese whispers graph clustering
// algorithm to identify how many people there are and which faces belong to whom.
var edges = new List <SamplePair>();
for (uint i = 0; i < faceDescriptors.Count; ++i)
{
for (var j = i; j < faceDescriptors.Count; ++j)
{
// Faces are connected in the graph if they are close enough. Here we check if
// the distance between two face descriptors is less than 0.6, which is the
// decision threshold the network was trained to use. Although you can
// certainly use any other threshold you find useful.
var diff = faceDescriptors[i] - faceDescriptors[j];
if (Dlib.Length(diff) < 0.6)
{
edges.Add(new SamplePair(i, j));
}
}
}
Dlib.ChineseWhispers(edges, 100, out var numClusters, out var labels);
// This will correctly indicate that there are 4 people in the image.
Console.WriteLine($"number of people found in the image: {numClusters}");
// Now let's display the face clustering results on the screen. You will see that it
// correctly grouped all the faces.
var winClusters = new List <ImageWindow>();
for (var i = 0; i < numClusters; i++)
{
winClusters.Add(new ImageWindow());
}
var tileImages = new List <Matrix <RgbPixel> >();
for (var clusterId = 0ul; clusterId < numClusters; ++clusterId)
{
var temp = new List <Matrix <RgbPixel> >();
for (var j = 0; j < labels.Length; ++j)
{
if (clusterId == labels[j])
{
temp.Add(faces[j]);
}
}
winClusters[(int)clusterId].Title = $"face cluster {clusterId}";
var tileImage = Dlib.TileImages(temp);
tileImages.Add(tileImage);
winClusters[(int)clusterId].SetImage(tileImage);
}
// Finally, let's print one of the face descriptors to the screen.
using (var trans = Dlib.Trans(faceDescriptors[0]))
{
Console.WriteLine($"face descriptor for one face: {trans}");
// It should also be noted that face recognition accuracy can be improved if jittering
// is used when creating face descriptors. In particular, to get 99.38% on the LFW
// benchmark you need to use the jitter_image() routine to compute the descriptors,
// like so:
var jitterImages = JitterImage(faces[0]).ToArray();
var ret = net.Operator(jitterImages);
using (var m = Dlib.Mat(ret))
using (var faceDescriptor = Dlib.Mean <float>(m))
using (var t = Dlib.Trans(faceDescriptor))
{
Console.WriteLine($"jittered face descriptor for one face: {t}");
// If you use the model without jittering, as we did when clustering the bald guys, it
// gets an accuracy of 99.13% on the LFW benchmark. So jittering makes the whole
// procedure a little more accurate but makes face descriptor calculation slower.
Console.WriteLine("hit enter to terminate");
Console.ReadKey();
foreach (var jitterImage in jitterImages)
{
jitterImage.Dispose();
}
foreach (var tileImage in tileImages)
{
tileImage.Dispose();
}
foreach (var edge in edges)
{
edge.Dispose();
}
foreach (var descriptor in faceDescriptors)
{
descriptor.Dispose();
}
foreach (var face in faces)
{
face.Dispose();
}
}
}
}
}
public SearchFaces(byte[] photoArray, int[] additionalRotateAngles = null, int photoJpegQuality = 95, double faceClippingRatio = 1.2)
{
if (photoArray != null)
{
try
{
using (Stream input = new MemoryStream(photoArray))
{
photoArray = null;
using (var inputStream = new SKManagedStream(input))
using (var codec = SKCodec.Create(inputStream))
using (var original = SKBitmap.Decode(codec))
{
// ЧИТАЕМ EXIF-ИНФОРМАЦИЮ
input.Position = 0;
try
{
using (ExifReader reader = new ExifReader(input))
{
reader.GetTagValue(ExifTags.DateTime, out photoDateTime);
reader.GetTagValue(ExifTags.BodySerialNumber, out cameraSerialNumber);
}
}
catch (Exception exc) { }
// НОРМАЛИЗУЕМ ИЗОБРАЖЕНИE ПО ВРАЩЕНИЮ
SKBitmap normalized = AdjustOrientation(original, codec.EncodedOrigin);
double scaleFactor = 2;
// ПОЛУЧАЕМ ДЕТЕКТИРУЕМОЕ НА ЛИЦА ИЗОБРАЖЕНИЕ
using (var scanned = normalized.Resize(new SKImageInfo((int)Math.Round((double)normalized.Width / scaleFactor), (int)Math.Round((double)normalized.Height / scaleFactor)), SKFilterQuality.High))
{
if (scanned == null)
{
return;
}
int additionalFacesCounter = 0;
List <FaceLocation> faceLocationList = new List <FaceLocation>();
using (var fd = Dlib.GetFrontalFaceDetector())
{
DlibDotNet.Rectangle[] faces = null;
using (var array2D = Dlib.LoadImageData <RgbPixel>(ImagePixelFormat.Rgba, scanned.Bytes, (uint)scanned.Height, (uint)scanned.Width, (uint)(scanned.Bytes.Length / scanned.Height)))
faces = fd.Operator(array2D);
if (faces != null && faces.Length > 0)
{
for (int f = 0; f < faces.Length; f++)
{
#region обрезаем лицо до квадрата
Point center = faces[f].Center;
int radius = 0;
if (faces[f].Width < faces[f].Height)
{
radius = (int)faces[f].Width / 2;
}
else
{
radius = (int)faces[f].Height / 2;
}
faces[f].Left = center.X - radius;
faces[f].Right = center.X + radius;
faces[f].Top = center.Y - radius;
faces[f].Bottom = center.Y + radius;
#endregion обрезаем лицо до квадрата
FaceLocation faceLocation = CalculateNormalFaceLocation(faces[f], normalized.Width, normalized.Height, scaleFactor, faceClippingRatio);
faceLocationList.Add(faceLocation);
}
}
if (additionalRotateAngles != null && additionalRotateAngles.Length > 0)
{
for (int r = 0; r < additionalRotateAngles.Length; r++)
{
if (additionalRotateAngles[r] != 0)
{
DlibDotNet.Rectangle[] addFaces = null;
SKBitmap rotatedScanned = Rotate(scanned, additionalRotateAngles[r]);
using (var array2D = Dlib.LoadImageData <RgbPixel>(ImagePixelFormat.Rgba, rotatedScanned.Bytes, (uint)rotatedScanned.Height, (uint)rotatedScanned.Width, (uint)(rotatedScanned.Bytes.Length / rotatedScanned.Height)))
addFaces = fd.Operator(array2D);
if (addFaces != null && addFaces.Length > 0)
{
for (int i = 0; i < addFaces.Length; i++)
{
#region обрезаем лицо до квадрата
Point center = addFaces[i].Center;
int radius = 0;
if (addFaces[i].Width < addFaces[i].Height)
{
radius = (int)addFaces[i].Width / 2;
}
else
{
radius = (int)addFaces[i].Height / 2;
}
addFaces[i].Left = center.X - radius;
addFaces[i].Right = center.X + radius;
addFaces[i].Top = center.Y - radius;
addFaces[i].Bottom = center.Y + radius;
#endregion обрезаем лицо до квадрата
FaceLocation faceLocation = CalculateRotatedFaceLocation((double)rotatedScanned.Width / 2, (double)rotatedScanned.Height / 2, addFaces[i], -additionalRotateAngles[r], normalized.Width, normalized.Height, scaleFactor, faceClippingRatio);
additionalFacesCounter++;
faceLocationList.Add(faceLocation);
}
}
}
}
}
}
if (faceLocationList.Count > 0)
{
List <FaceLocation> correlatedFaceList = GetCorrelatedFaceList(faceLocationList, additionalFacesCounter); // пропускаем через коррелятор лиц для избавления от дублей и уменьшения бокового наклона
if (correlatedFaceList != null && correlatedFaceList.Count > 0)
{
for (int i = 0; i < correlatedFaceList.Count; i++)
{
//var cropRect = new SKRectI { Left = correlatedFaceList[i].Left, Top = correlatedFaceList[i].Top, Right = correlatedFaceList[i].Right, Bottom = correlatedFaceList[i].Bottom };
var cropRect = new SKRectI();
int w = correlatedFaceList[i].Right - correlatedFaceList[i].Left;
int h = correlatedFaceList[i].Bottom - correlatedFaceList[i].Top;
int centerX = correlatedFaceList[i].Left + w / 2;
int centerY = correlatedFaceList[i].Top + h / 2;
if (w > h)
{
cropRect.Left = centerX - h / 2;
cropRect.Right = centerX + h / 2;
cropRect.Top = centerY - h / 2;
cropRect.Bottom = centerY + h / 2;
}
else if (w < h)
{
cropRect.Left = centerX - w / 2;
cropRect.Right = centerX + w / 2;
cropRect.Top = centerY - w / 2;
cropRect.Bottom = centerY + w / 2;
}
else
{
cropRect.Left = correlatedFaceList[i].Left;
cropRect.Top = correlatedFaceList[i].Top;
cropRect.Right = correlatedFaceList[i].Right;
cropRect.Bottom = correlatedFaceList[i].Bottom;
}
var faceBitmap = new SKBitmap(cropRect.Width, cropRect.Height);
normalized.ExtractSubset(faceBitmap, cropRect);
//// ТЕПЕРЬ БУДЕМ ПОВОРАЧИВАТЬ
SKBitmap rotated = Rotate(faceBitmap, -correlatedFaceList[i].Angle);
// ТЕПЕРЬ НАКЛАДЫВАЕМ МАСКУ НА ЛИЦО В ВИДЕ КРУГА
double radius = 0;
if (cropRect.Width < cropRect.Height)
{
radius = (double)cropRect.Width / 2 * (1 + 0.5 / 2);
}
else
{
radius = (double)cropRect.Height / 2 * (1 + 0.5 / 2);
}
using (SKCanvas canvas = new SKCanvas(rotated))
{
canvas.DrawBitmap(rotated, 0, 0);
SKPaint paint = new SKPaint();
paint.Color = maskColor;
paint.Style = SKPaintStyle.Stroke;
paint.StrokeWidth = (float)(radius * 0.4);
canvas.DrawCircle((float)rotated.Width / 2, (float)rotated.Height / 2, (float)radius, paint);
canvas.Flush();
}
// ВЫРЕЗАЕМ ИТОГ
double x = (double)rotated.Width / 2;
double y = (double)rotated.Height / 2;
var finalCropRect = new SKRectI {
Left = (int)(x - (double)faceBitmap.Width / 2), Top = (int)(y - (double)faceBitmap.Height / 2), Right = (int)(x + (double)faceBitmap.Width / 2), Bottom = (int)(y + (double)faceBitmap.Height / 2)
};
faceBitmap.Dispose();
using (SKBitmap face = new SKBitmap(finalCropRect.Width, finalCropRect.Height))
{
rotated.ExtractSubset(face, finalCropRect);
try
{
if (face.Width > 600 * scaleFactor)
{
using (var scaled = face.Resize(new SKImageInfo((int)Math.Round(400 * scaleFactor), (int)Math.Round(400 * scaleFactor)), SKFilterQuality.High))
{
if (scaled != null)
{
using (var image = SKImage.FromBitmap(scaled))
using (var data = image.Encode(SKEncodedImageFormat.Jpeg, 90))
{
if (facesByteArrays == null)
{
facesByteArrays = new List <byte[]>();
}
facesByteArrays.Add(data.ToArray());
}
}
}
}
else
{
using (var image = SKImage.FromBitmap(face))
using (var data = image.Encode(SKEncodedImageFormat.Jpeg, 90))
{
if (facesByteArrays == null)
{
facesByteArrays = new List <byte[]>();
}
facesByteArrays.Add(data.ToArray());
}
}
}
catch (Exception exc) { };
}
}
normalized.Dispose();
correlatedFaceList = null;
}
faceLocationList = null;
}
}
}
isSuccess = true;
}
}
catch (Exception exc)
{
try
{
isSuccess = false;
if (exc.StackTrace != null && exc.StackTrace != "")
{
log.Debug(String.Format("SearchFaces. Exception: {0}", exc.StackTrace));
}
else if (exc.Message != null && exc.Message != "")
{
log.Debug(String.Format("SearchFaces. Exception: {0}", exc.Message));
}
}
catch (Exception ex) { }
}
}
else
{
log.Debug("SearchFaces. Null request.");
}
}
