public List <List <int> > ClusterFaces(MatrixFloatDto[] faces)
{
var convertedFaces = faces
.Select(d => new Matrix <float>(d.Data, d.Row, d.Columns))
.ToArray();
// compare each face with all other faces
var edges = new List <SamplePair>();
for (var i = 0; i < convertedFaces.Length; ++i)
{
for (var j = i; j < convertedFaces.Length; ++j)
{
// record every pair of two similar faces
// faces are similar if they are less than 0.6 apart in the 128D embedding space
if (Dlib.Length(convertedFaces[i] - convertedFaces[j]) < 0.4)
{
edges.Add(new SamplePair((uint)i, (uint)j));
}
}
}
// use the chinese whispers algorithm to find all face clusters
Dlib.ChineseWhispers(edges, 100, out var foundClusterCount, out var outputLabels);
var clusters = new List <List <int> >((int)foundClusterCount);
for (var i = 0; i < foundClusterCount; i++)
{
clusters.Add(new List <int>());
}
for (var index = 0; index < faces.Length; index++)
{
var foundCluster = (int)outputLabels[index];
clusters[foundCluster].Add(index);
}
return(clusters);
}
private static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Run this example by invoking it like this: ");
Console.WriteLine(" ./DnnFaceRecognition faces/bald_guys.jpg");
Console.WriteLine("You will also need to get the face landmarking model file as well as ");
Console.WriteLine("the face recognition model file. Download and then decompress these files from: ");
Console.WriteLine("http://dlib.net/files/shape_predictor_5_face_landmarks.dat.bz2");
Console.WriteLine("http://dlib.net/files/dlib_face_recognition_resnet_model_v1.dat.bz2");
return;
}
// The first thing we are going to do is load all our models. First, since we need to
// find faces in the image we will need a face detector:
using (var detector = FrontalFaceDetector.GetFrontalFaceDetector())
// We will also use a face landmarking model to align faces to a standard pose: (see face_landmark_detection_ex.cpp for an introduction)
using (var sp = new ShapePredictor("shape_predictor_5_face_landmarks.dat"))
// And finally we load the DNN responsible for face recognition.
using (var net = DlibDotNet.Dnn.LossMetric.Deserialize("dlib_face_recognition_resnet_model_v1.dat"))
using (var img = Dlib.LoadImage <RgbPixel>(args[0]))
using (var mat = new Matrix <RgbPixel>(img))
// Display the raw image on the screen
using (var win = new ImageWindow(img))
{
// Run the face detector on the image of our action heroes, and for each face extract a
// copy that has been normalized to 150x150 pixels in size and appropriately rotated
// and centered.
var faces = new List <Matrix <RgbPixel> >();
foreach (var face in detector.Detect(img))
{
var shape = sp.Detect(img, face);
var faceChipDetail = Dlib.GetFaceChipDetails(shape, 150, 0.25);
var faceChip = Dlib.ExtractImageChip <RgbPixel>(mat, faceChipDetail);
//faces.Add(move(face_chip));
faces.Add(faceChip);
// Also put some boxes on the faces so we can see that the detector is finding
// them.
win.AddOverlay(face);
}
if (!faces.Any())
{
Console.WriteLine("No faces found in image!");
return;
}
// This call asks the DNN to convert each face image in faces into a 128D vector.
// In this 128D vector space, images from the same person will be close to each other
// but vectors from different people will be far apart. So we can use these vectors to
// identify if a pair of images are from the same person or from different people.
var faceDescriptors = net.Operator(faces);
// In particular, one simple thing we can do is face clustering. This next bit of code
// creates a graph of connected faces and then uses the Chinese whispers graph clustering
// algorithm to identify how many people there are and which faces belong to whom.
var edges = new List <SamplePair>();
for (uint i = 0; i < faceDescriptors.Count; ++i)
{
for (var j = i; j < faceDescriptors.Count; ++j)
{
// Faces are connected in the graph if they are close enough. Here we check if
// the distance between two face descriptors is less than 0.6, which is the
// decision threshold the network was trained to use. Although you can
// certainly use any other threshold you find useful.
var diff = faceDescriptors[i] - faceDescriptors[j];
if (Dlib.Length(diff) < 0.6)
{
edges.Add(new SamplePair(i, j));
}
}
}
Dlib.ChineseWhispers(edges, 100, out var numClusters, out var labels);
// This will correctly indicate that there are 4 people in the image.
Console.WriteLine($"number of people found in the image: {numClusters}");
// Now let's display the face clustering results on the screen. You will see that it
// correctly grouped all the faces.
var winClusters = new List <ImageWindow>();
for (var i = 0; i < numClusters; i++)
{
winClusters.Add(new ImageWindow());
}
var tileImages = new List <Matrix <RgbPixel> >();
for (var clusterId = 0ul; clusterId < numClusters; ++clusterId)
{
var temp = new List <Matrix <RgbPixel> >();
for (var j = 0; j < labels.Length; ++j)
{
if (clusterId == labels[j])
{
temp.Add(faces[j]);
}
}
winClusters[(int)clusterId].Title = $"face cluster {clusterId}";
var tileImage = Dlib.TileImages(temp);
tileImages.Add(tileImage);
winClusters[(int)clusterId].SetImage(tileImage);
}
// Finally, let's print one of the face descriptors to the screen.
using (var trans = Dlib.Trans(faceDescriptors[0]))
{
Console.WriteLine($"face descriptor for one face: {trans}");
// It should also be noted that face recognition accuracy can be improved if jittering
// is used when creating face descriptors. In particular, to get 99.38% on the LFW
// benchmark you need to use the jitter_image() routine to compute the descriptors,
// like so:
var jitterImages = JitterImage(faces[0]).ToArray();
var ret = net.Operator(jitterImages);
using (var m = Dlib.Mat(ret))
using (var faceDescriptor = Dlib.Mean <float>(m))
using (var t = Dlib.Trans(faceDescriptor))
{
Console.WriteLine($"jittered face descriptor for one face: {t}");
// If you use the model without jittering, as we did when clustering the bald guys, it
// gets an accuracy of 99.13% on the LFW benchmark. So jittering makes the whole
// procedure a little more accurate but makes face descriptor calculation slower.
Console.WriteLine("hit enter to terminate");
Console.ReadKey();
foreach (var jitterImage in jitterImages)
{
jitterImage.Dispose();
}
foreach (var tileImage in tileImages)
{
tileImage.Dispose();
}
foreach (var edge in edges)
{
edge.Dispose();
}
foreach (var descriptor in faceDescriptors)
{
descriptor.Dispose();
}
foreach (var face in faces)
{
face.Dispose();
}
}
}
}
}
public static IEnumerable <Matches> ClusterFacesByPosition(List <System.Drawing.Rectangle> set1, List <System.Drawing.Rectangle> set2, List <System.Drawing.Rectangle> set3)
{
var totalset = set1.Concat(set2).Concat(set3).ToList();
var matches = new List <SamplePair>();
for (var i = 0; i < totalset.Count; i++)
{
matches.Add(new SamplePair((uint)i, (uint)i, 0d));
}
for (var i = 0; i < set1.Count; i++)
{
for (var j = set1.Count; j < totalset.Count; j++)
{
var intersection = System.Drawing.Rectangle.Intersect(totalset[i], totalset[j]);
if (intersection.IsEmpty)
{
continue;
}
var intersectionSize = intersection.Width * intersection.Height;
var unionSize = totalset[i].Width * totalset[i].Height;
unionSize += (totalset[j].Width * totalset[j].Height);
unionSize -= intersectionSize;
var iou = (double)intersectionSize / (double)unionSize;
if (iou > 0.45)
{
matches.Add(new SamplePair((uint)i, (uint)(j), 1 - iou));
}
}
}
for (var i = set1.Count; i < set1.Count + set2.Count; i++)
{
for (var j = set1.Count + set2.Count; j < totalset.Count; j++)
{
var intersection = System.Drawing.Rectangle.Intersect(totalset[i], totalset[j]);
if (intersection.IsEmpty)
{
continue;
}
var intersectionSize = intersection.Width * intersection.Height;
var unionSize = totalset[i].Width * totalset[i].Height;
unionSize += (totalset[j].Width * totalset[j].Height);
unionSize -= intersectionSize;
var iou = (double)intersectionSize / (double)unionSize;
if (iou > 0.45)
{
matches.Add(new SamplePair((uint)i, (uint)(j), 1 - iou));
}
}
}
// use the chinese whispers algorithm to find all face clusters
Dlib.ChineseWhispers(matches, 100, out var clusters2, out var labels2);
var result = new List <Matches>((int)clusters2);
for (int i = 0; i < clusters2; i++)
{
result.Add(new Matches());
}
for (int i = 0; i < set1.Count; i++)
{
var j = i;
var label = (int)labels2[j];
result[label].Set(1, i);
}
for (int i = 0; i < set2.Count; i++)
{
var j = i + set1.Count;
var label = (int)labels2[j];
result[label].Set(2, i);
}
for (int i = 0; i < set3.Count; i++)
{
var j = i + set1.Count + set2.Count;
var label = (int)labels2[j];
result[label].Set(3, i);
}
return(result);
}
public async Task ProcessAsync(string[] inputFilenames)
{
var chips = new List <Matrix <RgbPixel> >();
var faces = new List <Rectangle>();
var filename = new List <string>();
var jsonFilename = inputFilenames.First() + ".json";
foreach (var inputFilename in inputFilenames)
{
if (!File.Exists(inputFilename))
{
break;
}
if (File.Exists(jsonFilename))
{
continue;
}
// load the image
using var img = await DlibHelpers.LoadRotatedImage(imageRotationService, inputFilename);
// Dlib.SaveJpeg(img, inputFilename + "__1.jpg", 25);
// Dlib.SaveJpeg(img, inputFilename + "__2.jpg", 25);
// detect all faces
foreach (var face in detector.Operator(img))
{
// detect landmarks
var shape = predictor.Detect(img, face);
// extract normalized and rotated 150x150 face chip
var faceChipDetail = Dlib.GetFaceChipDetails(shape, 150, 0.25);
var faceChip = Dlib.ExtractImageChip <RgbPixel>(img, faceChipDetail);
// convert the chip to a matrix and store
var matrix = new Matrix <RgbPixel>(faceChip);
chips.Add(matrix);
faces.Add(face);
filename.Add(inputFilename);
}
}
if (!File.Exists(jsonFilename))
{
var ffd = new FoundFacesData
{
// Chips = chips,
Faces = faces,
Filenames = filename,
};
OutputLabels <Matrix <float> > descriptors = null;
if (chips.Any())
{
// put each fae in a 128D embedding space
// similar faces will be placed close together
// Console.WriteLine("Recognizing faces...");
descriptors = dnn.Operator(chips);
ffd.Descriptors = descriptors.ToList();
}
else
{
ffd.Descriptors = new List <Matrix <float> >(0);
}
var dto = new FoundFacesDataDto
{
Faces = ffd.Faces
.Select(f => new RectangleDto
{
Bottom = f.Bottom,
Left = f.Left,
Top = f.Top,
Right = f.Right,
})
.ToList(),
Filenames = ffd.Filenames,
Descriptors = ffd.Descriptors
.Select(x => new MatrixFloatDto
{
Data = x.ToArray(),
Row = x.Rows,
Columns = x.Columns,
})
.ToList()
};
var x = JsonConvert.SerializeObject(dto);
File.WriteAllText(jsonFilename, JsonConvert.SerializeObject(dto));
}
FoundFacesData items;
using (var r = new StreamReader(jsonFilename))
{
var json = r.ReadToEnd();
var itemsdto = JsonConvert.DeserializeObject <FoundFacesDataDto>(json);
items = new FoundFacesData
{
Faces = itemsdto.Faces.Select(f => new Rectangle(f.Left, f.Top, f.Right, f.Bottom)).ToList(),
Filenames = itemsdto.Filenames.ToList(),
Descriptors = itemsdto.Descriptors.Select(d => new Matrix <float>(d.Data, d.Row, d.Columns)).ToList(),
};
}
if (items.Faces.Count <= 0)
{
return;
}
// // compare each face with all other faces
var edges = new List <SamplePair>();
// for (uint i = 0; i < descriptors.Count; ++i)
// for (var j = i; j < descriptors.Count; ++j)
//
// // record every pair of two similar faces
// // faces are similar if they are less than 0.6 apart in the 128D embedding space
// if (Dlib.Length(descriptors[i] - descriptors[j]) < 0.5)
// edges.Add(new SamplePair(i, j));
//
// // use the chinese whispers algorithm to find all face clusters
// Dlib.ChineseWhispers(edges, 100, out var clusters, out var labels);
// // Console.WriteLine($" Found {clusters} unique person(s) in the image");
//
// // draw rectangles on each face using the cluster color
// for (var i = 0; i < faces.Count; i++)
// {
// var color = new RgbPixel(255, 255, 255);
// if (labels[i] < palette.Length)
// color = palette[labels[i]];
//
// using var img = Dlib.LoadImage<RgbPixel>(filename[i] + "__1.jpg");
// Dlib.DrawRectangle(img, faces[i], color: color, thickness: 4);
// Dlib.SaveJpeg(img, filename[i] + "__1.jpg", 25);
// }
//
// Console.WriteLine("end 1");
// compare each face with all other faces
edges = new List <SamplePair>();
for (var i = 0; i < items.Descriptors.Count; ++i)
{
for (var j = i; j < items.Descriptors.Count; ++j)
{
// record every pair of two similar faces
// faces are similar if they are less than 0.6 apart in the 128D embedding space
if (Dlib.Length(items.Descriptors[i] - items.Descriptors[j]) < 0.4)
{
edges.Add(new SamplePair((uint)i, (uint)j));
}
}
}
// use the chinese whispers algorithm to find all face clusters
Dlib.ChineseWhispers(edges, 100, out var clusters2, out var labels2);
// Console.WriteLine($" Found {clusters} unique person(s) in the image");
// draw rectangles on each face using the cluster color
for (var i = 0; i < items.Faces.Count; i++)
{
var color = palette[0];
if (labels2[i] < palette.Length)
{
color = palette[labels2[i]];
}
if (!File.Exists(items.Filenames[i] + $"_x{labels2[i]}.jpg"))
{
using var img2 = await DlibHelpers.LoadRotatedImage(imageRotationService, items.Filenames[i]);
Dlib.SaveJpeg(img2, items.Filenames[i] + $"_x{labels2[i]}.jpg", 25);
}
using var img = Dlib.LoadImage <RgbPixel>(items.Filenames[i] + $"_x{labels2[i]}.jpg");
Dlib.DrawRectangle(img, items.Faces[i], color: color, thickness: 4);
Dlib.SaveJpeg(img, items.Filenames[i] + $"_x{labels2[i]}.jpg", 25);
}
// var origFilename = new FileInfo(inputFilename).Name;
// var outputFilename = Path.Combine(outputDirectory, $"{origFilename}_Identification.jpg");
// Dlib.SaveJpeg(img, inputFilename, 75);
}