private void RecordObservation(RecorderOpenFace recorder, RawImage vis_image, int face_id, bool success, float fx, float fy, float cx, float cy, double timestamp, int frame_number)
{
recorder.SetObservationTimestamp(timestamp);
double confidence = landmark_detector.GetConfidence();
List <float> pose = new List <float>();
landmark_detector.GetPose(pose, fx, fy, cx, cy);
recorder.SetObservationPose(pose);
List <Tuple <float, float> > landmarks_2D = landmark_detector.CalculateAllLandmarks();
List <Tuple <float, float, float> > landmarks_3D = landmark_detector.Calculate3DLandmarks(fx, fy, cx, cy);
List <float> global_params = landmark_detector.GetRigidParams();
List <float> local_params = landmark_detector.GetNonRigidParams();
recorder.SetObservationLandmarks(landmarks_2D, landmarks_3D, global_params, local_params, confidence, success);
var gaze = gaze_analyser.GetGazeCamera();
var gaze_angle = gaze_analyser.GetGazeAngle();
var landmarks_2d_eyes = landmark_detector.CalculateAllEyeLandmarks();
var landmarks_3d_eyes = landmark_detector.CalculateAllEyeLandmarks3D(fx, fy, cx, cy);
recorder.SetObservationGaze(gaze.Item1, gaze.Item2, gaze_angle, landmarks_2d_eyes, landmarks_3d_eyes);
var au_regs = face_analyser.GetCurrentAUsReg();
var au_classes = face_analyser.GetCurrentAUsClass();
recorder.SetObservationActionUnits(au_regs, au_classes);
recorder.SetObservationFaceID(face_id);
recorder.SetObservationFrameNumber(frame_number);
recorder.SetObservationFaceAlign(face_analyser.GetLatestAlignedFace());
var hog_feature = face_analyser.GetLatestHOGFeature();
recorder.SetObservationHOG(success, hog_feature, face_analyser.GetHOGRows(), face_analyser.GetHOGCols(), face_analyser.GetHOGChannels());
recorder.SetObservationVisualization(vis_image);
recorder.WriteObservation();
}
// The main function call for processing the webcam feed
private void ProcessingLoop(SequenceReader reader)
{
thread_running = true;
Thread.CurrentThread.IsBackground = true;
DateTime?startTime = CurrentTime;
var lastFrameTime = CurrentTime;
landmark_detector.Reset();
face_analyser.Reset();
int frame_id = 0;
double old_gaze_x = 0;
double old_gaze_y = 0;
double smile_cumm = 0;
double frown_cumm = 0;
double brow_up_cumm = 0;
double brow_down_cumm = 0;
double widen_cumm = 0;
double wrinkle_cumm = 0;
while (thread_running)
{
// Loading an image file
RawImage frame = new RawImage(reader.GetNextImage());
RawImage gray_frame = new RawImage(reader.GetCurrentFrameGray());
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(gray_frame, face_model_params);
// The face analysis step (only done if recording AUs, HOGs or video)
face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, true);
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
double confidence = landmark_detector.GetConfidence();
if (confidence < 0)
{
confidence = 0;
}
else if (confidence > 1)
{
confidence = 1;
}
List <double> pose = new List <double>();
landmark_detector.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
List <double> non_rigid_params = landmark_detector.GetNonRigidParams();
double scale = landmark_detector.GetRigidParams()[0];
double time_stamp = (DateTime.Now - (DateTime)startTime).TotalMilliseconds;
List <Tuple <Point, Point> > lines = null;
List <Tuple <double, double> > landmarks = null;
List <Tuple <double, double> > eye_landmarks = null;
List <Tuple <Point, Point> > gaze_lines = null;
Tuple <double, double> gaze_angle = gaze_analyser.GetGazeAngle();
if (detection_succeeding)
{
landmarks = landmark_detector.CalculateVisibleLandmarks();
eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks();
lines = landmark_detector.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
}
// Visualisation
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
var au_regs = face_analyser.GetCurrentAUsReg();
if (au_regs.Count > 0)
{
double smile = (au_regs["AU12"] + au_regs["AU06"] + au_regs["AU25"]) / 13.0;
double frown = (au_regs["AU15"] + au_regs["AU17"]) / 12.0;
double brow_up = (au_regs["AU01"] + au_regs["AU02"]) / 10.0;
double brow_down = au_regs["AU04"] / 5.0;
double eye_widen = au_regs["AU05"] / 3.0;
double nose_wrinkle = au_regs["AU09"] / 4.0;
Dictionary <int, double> smileDict = new Dictionary <int, double>();
smileDict[0] = 0.7 * smile_cumm + 0.3 * smile;
smileDict[1] = 0.7 * frown_cumm + 0.3 * frown;
smilePlot.AddDataPoint(new DataPointGraph()
{
Time = CurrentTime, values = smileDict, Confidence = confidence
});
Dictionary <int, double> browDict = new Dictionary <int, double>();
browDict[0] = 0.7 * brow_up_cumm + 0.3 * brow_up;
browDict[1] = 0.7 * brow_down_cumm + 0.3 * brow_down;
browPlot.AddDataPoint(new DataPointGraph()
{
Time = CurrentTime, values = browDict, Confidence = confidence
});
Dictionary <int, double> eyeDict = new Dictionary <int, double>();
eyeDict[0] = 0.7 * widen_cumm + 0.3 * eye_widen;
eyeDict[1] = 0.7 * wrinkle_cumm + 0.3 * nose_wrinkle;
eyePlot.AddDataPoint(new DataPointGraph()
{
Time = CurrentTime, values = eyeDict, Confidence = confidence
});
smile_cumm = smileDict[0];
frown_cumm = smileDict[1];
brow_up_cumm = browDict[0];
brow_down_cumm = browDict[1];
widen_cumm = eyeDict[0];
wrinkle_cumm = eyeDict[1];
}
else
{
// If no AUs present disable the AU visualization
MainGrid.ColumnDefinitions[2].Width = new GridLength(0);
eyePlot.Visibility = Visibility.Collapsed;
browPlot.Visibility = Visibility.Collapsed;
smilePlot.Visibility = Visibility.Collapsed;
}
Dictionary <int, double> poseDict = new Dictionary <int, double>();
poseDict[0] = -pose[3];
poseDict[1] = pose[4];
poseDict[2] = pose[5];
headPosePlot.AddDataPoint(new DataPointGraph()
{
Time = CurrentTime, values = poseDict, Confidence = confidence
});
Dictionary <int, double> gazeDict = new Dictionary <int, double>();
gazeDict[0] = gaze_angle.Item1 * (180.0 / Math.PI);
gazeDict[0] = 0.5 * old_gaze_x + 0.5 * gazeDict[0];
gazeDict[1] = -gaze_angle.Item2 * (180.0 / Math.PI);
gazeDict[1] = 0.5 * old_gaze_y + 0.5 * gazeDict[1];
gazePlot.AddDataPoint(new DataPointGraph()
{
Time = CurrentTime, values = gazeDict, Confidence = confidence
});
old_gaze_x = gazeDict[0];
old_gaze_y = gazeDict[1];
if (latest_img == null)
{
latest_img = frame.CreateWriteableBitmap();
}
frame.UpdateWriteableBitmap(latest_img);
video.Source = latest_img;
video.Confidence = confidence;
video.FPS = processing_fps.GetFPS();
if (!detection_succeeding)
{
video.OverlayLines.Clear();
video.OverlayPoints.Clear();
video.OverlayEyePoints.Clear();
video.GazeLines.Clear();
}
else
{
video.OverlayLines = lines;
List <Point> landmark_points = new List <Point>();
foreach (var p in landmarks)
{
landmark_points.Add(new Point(p.Item1, p.Item2));
}
List <Point> eye_landmark_points = new List <Point>();
foreach (var p in eye_landmarks)
{
eye_landmark_points.Add(new Point(p.Item1, p.Item2));
}
video.OverlayPoints = landmark_points;
video.OverlayEyePoints = eye_landmark_points;
video.GazeLines = gaze_lines;
}
}));
if (reset)
{
if (resetPoint.HasValue)
{
landmark_detector.Reset(resetPoint.Value.X, resetPoint.Value.Y);
resetPoint = null;
}
else
{
landmark_detector.Reset();
}
face_analyser.Reset();
reset = false;
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
headPosePlot.ClearDataPoints();
headPosePlot.ClearDataPoints();
gazePlot.ClearDataPoints();
smilePlot.ClearDataPoints();
browPlot.ClearDataPoints();
eyePlot.ClearDataPoints();
}));
}
frame_id++;
}
reader.Close();
latest_img = null;
}