private Bitmap ProcessImage(SequenceReader reader)
{
// set up the face model
var root = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"..\..\");
var faceModel = new FaceModelParameters(root, false);
faceModel.optimiseForImages();
// set up a face detector, a landmark detector, and a face analyser
var faceDetector = new FaceDetector();
var landmarkDetector = new CLNF(faceModel);
var faceAnalyser = new FaceAnalyserManaged(root, true, 0);
// read the image from the sequence reader
var frame = new RawImage(reader.GetNextImage());
var grayFrame = new RawImage(reader.GetCurrentFrameGray());
// detect faces
var faces = new List <Rect>();
var confidences = new List <double>();
faceDetector.DetectFacesHOG(faces, grayFrame, confidences);
// detect landmarks
var image = frame.ToBitmap();
foreach (var face in faces)
{
landmarkDetector.DetectFaceLandmarksInImage(grayFrame, face, faceModel);
var points = landmarkDetector.CalculateAllLandmarks();
// calculate action units
var features = faceAnalyser.PredictStaticAUsAndComputeFeatures(grayFrame, points);
// find the action units
var actionUnits = (from au in features.Item2
where au.Value > 0
orderby au.Key
select au.Key);
// get top emotions
var topEmotions = GetTopEmotions(actionUnits);
// draw the emotion on the face
using (Graphics g = Graphics.FromImage(image))
{
string name = string.Join(Environment.NewLine, topEmotions);
Font fnt = new Font("Verdana", 15, GraphicsUnit.Pixel);
Brush brs = new SolidBrush(Color.Black);
var bump = 36;
System.Drawing.SizeF stringSize = g.MeasureString(name, fnt);
g.FillRectangle(new SolidBrush(Color.Yellow), (int)face.X + bump, (int)face.Y, stringSize.Width, stringSize.Height);
g.DrawString(name, fnt, brs, (int)face.X + bump, (int)face.Y);
}
}
return(image);
}
private Bitmap ProcessImage(SequenceReader reader)
{
// set up the face model
String root = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"..\..\");
var faceModel = new FaceModelParameters(root, false);
faceModel.optimiseForImages();
// set up a face detector and a landmark detector
var faceDetector = new FaceDetector();
var landmarkDetector = new CLNF(faceModel);
// read the image from the sequence reader
var frame = new RawImage(reader.GetNextImage());
var grayFrame = new RawImage(reader.GetCurrentFrameGray());
// detect faces
var faces = new List <Rect>();
var confidences = new List <double>();
faceDetector.DetectFacesHOG(faces, grayFrame, confidences);
// detect landmarks
var landmarks = new List <List <Tuple <double, double> > >();
foreach (var face in faces)
{
landmarkDetector.DetectFaceLandmarksInImage(grayFrame, face, faceModel);
var points = landmarkDetector.CalculateAllLandmarks();
landmarks.Add(points);
}
// draw rectangles and confidence values on image
var image = frame.ToBitmap();
using (Graphics g = Graphics.FromImage(image))
{
int index = 0;
var pen = new System.Drawing.Pen(System.Drawing.Color.LightGreen, 4);
var pen2 = new System.Drawing.Pen(System.Drawing.Color.Red, 4);
var font = new Font(FontFamily.GenericSansSerif, 30);
foreach (var face in faces)
{
g.DrawRectangle(pen, (int)face.X, (int)face.Y, (int)face.Width, (int)face.Height);
g.DrawString($"{confidences[index]:0.00}", font, Brushes.Black, (int)face.X + 36, (int)face.Y - 36);
// draw landmark points
foreach (var p in landmarks[index])
{
g.DrawRectangle(pen2, new Rectangle((int)p.Item1, (int)p.Item2, 4, 4));
}
index++;
}
}
return(image);
}
/// <summary>
/// Calculate the current gaze point.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
static void OnGazeMove(object sender, GazeEventArgs e)
{
// read the image from the sequence reader
var frame = new RawImage(reader.GetNextImage());
var grayFrame = new RawImage(reader.GetCurrentFrameGray());
// only detect faces every 5 frames
if (currentFace == default(Rect) || frameIndex % 5 == 0)
{
// detect faces
var faces = new List <System.Windows.Rect>();
var confidences = new List <double>();
faceDetector.DetectFacesHOG(faces, grayFrame, confidences);
// abort if we couldn't find a face
if (currentFace == default(Rect) && faces.Count() == 0)
{
return;
}
currentFace = faces.First();
}
// detect landmarks
var success = landmarkDetector.DetectLandmarksInVideo(grayFrame, faceModel);
// calculate gaze angle
gazeAnalyser.AddNextFrame(landmarkDetector, success, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
var gaze = gazeAnalyser.GetGazeAngle();
// convert gaze to a point coordinate
e.Point.X = (int)(5000 * Math.Tan(-gaze.Item1) + form.Width / 2);
e.Point.Y = form.Height / 2;
// update frame counter
frameIndex++;
}
private void ProcessIndividualImages(ImageReader reader)
{
// Make sure the GUI is setup appropriately
SetupFeatureExtractionMode();
// Indicate we will start running the thread
thread_running = true;
// Reload the face landmark detector if needed
ReloadLandmarkDetector();
if (!landmark_detector.isLoaded())
{
DetectorNotFoundWarning();
EndMode();
thread_running = false;
return;
}
// Setup the parameters optimized for working on individual images rather than sequences
face_model_params.optimiseForImages();
// Setup the visualization
Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance, false);
// Initialize the face detector if it has not been initialized yet
if (face_detector == null)
{
face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());
}
// Initialize the face analyser
face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, false, image_output_size, MaskAligned);
// Loading an image file
var frame = new RawImage(reader.GetNextImage());
var gray_frame = new RawImage(reader.GetCurrentFrameGray());
// For FPS tracking
DateTime?startTime = CurrentTime;
var lastFrameTime = CurrentTime;
// This will be false when the image is not available
while (reader.isOpened())
{
if (!thread_running)
{
break;
}
// Setup recording
RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false,
Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs,
RecordGaze, RecordHOG, RecordTracked, RecordAligned, true,
reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0);
RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root);
// Detect faces here and return bounding boxes
List <Rect> face_detections = new List <Rect>();
List <float> confidences = new List <float>();
if (DetectorHOG)
{
face_detector.DetectFacesHOG(face_detections, gray_frame, confidences);
}
else if (DetectorCNN)
{
face_detector.DetectFacesMTCNN(face_detections, frame, confidences);
}
else if (DetectorHaar)
{
face_detector.DetectFacesHaar(face_detections, gray_frame, confidences);
}
// For visualization
double progress = reader.GetProgress();
for (int i = 0; i < face_detections.Count; ++i)
{
bool detection_succeeding = landmark_detector.DetectFaceLandmarksInImage(frame, face_detections[i], face_model_params, gray_frame);
var landmarks = landmark_detector.CalculateAllLandmarks();
// Predict action units
var au_preds = face_analyser.PredictStaticAUsAndComputeFeatures(frame, landmarks);
// Predic eye gaze
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
// Only the final face will contain the details
VisualizeFeatures(frame, visualizer_of, landmarks, landmark_detector.GetVisibilities(), detection_succeeding, i == 0, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress);
// Record an observation
RecordObservation(recorder, visualizer_of.GetVisImage(), i, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0, 0);
}
frame = new RawImage(reader.GetNextImage());
gray_frame = new RawImage(reader.GetCurrentFrameGray());
// Write out the tracked image
if (RecordTracked)
{
recorder.WriteObservationTracked();
}
// Do not cary state accross images
landmark_detector.Reset();
face_analyser.Reset();
recorder.Close();
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
// TODO how to report errors from the reader here? exceptions? logging? Problem for future versions?
}
EndMode();
}