public MainWindow()
{
InitializeComponent();
this.DataContext = this; // For WPF data binding
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
this.Icon = BitmapFrame.Create(iconUri);
String root = AppDomain.CurrentDomain.BaseDirectory;
face_model_params = new FaceModelParameters(root, LandmarkDetectorCECLM, LandmarkDetectorCLNF, LandmarkDetectorCLM);
// Initialize the face detector
face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());
// If MTCNN model not available, use HOG
if (!face_detector.IsMTCNNLoaded())
{
FaceDetCNN.IsEnabled = false;
DetectorCNN = false;
DetectorHOG = true;
}
face_model_params.SetFaceDetector(DetectorHaar, DetectorHOG, DetectorCNN);
landmark_detector = new CLNF(face_model_params);
gaze_analyser = new GazeAnalyserManaged();
}
private void Initialize(object sender, PipelineRunEventArgs e)
{
string rootDirectory = AppDomain.CurrentDomain.BaseDirectory;
faceModelParameters = new FaceModelParameters(rootDirectory, true, false, false);
faceModelParameters.optimiseForVideo();
faceDetector = new FaceDetector(faceModelParameters.GetHaarLocation(), faceModelParameters.GetMTCNNLocation());
if (!faceDetector.IsMTCNNLoaded())
{
faceModelParameters.SetFaceDetector(false, true, false);
}
landmarkDetector = new CLNF(faceModelParameters);
faceAnalyser = new FaceAnalyser(rootDirectory, dynamic: true, output_width: 112, mask_aligned: true);
gazeAnalyser = new GazeAnalyser();
landmarkDetector.Reset();
faceAnalyser.Reset();
}
protected ExtractorBase(FaceModelParameters faceModelParameters)
{
if (_initialized)
{
return;
}
ModelParams = faceModelParameters;
GazeAnalyzer = new GazeAnalyserManaged();
var face_detector = new FaceDetector(ModelParams.GetHaarLocation(), ModelParams.GetMTCNNLocation());
if (!face_detector.IsMTCNNLoaded()) // If MTCNN model not available, use HOG
{
ModelParams.SetFaceDetector(false, true, false);
}
FaceModel = new CLNF(ModelParams);
_initialized = true;
}
private void ProcessIndividualImages(ImageReader reader)
{
// Make sure the GUI is setup appropriately
SetupFeatureExtractionMode();
// Indicate we will start running the thread
thread_running = true;
// Reload the face landmark detector if needed
ReloadLandmarkDetector();
if (!landmark_detector.isLoaded())
{
DetectorNotFoundWarning();
EndMode();
thread_running = false;
return;
}
// Setup the parameters optimized for working on individual images rather than sequences
face_model_params.optimiseForImages();
// Setup the visualization
Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance, false);
// Initialize the face detector if it has not been initialized yet
if (face_detector == null)
{
face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());
}
// Initialize the face analyser
face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, false, image_output_size, MaskAligned);
// Loading an image file
var frame = new RawImage(reader.GetNextImage());
var gray_frame = new RawImage(reader.GetCurrentFrameGray());
// For FPS tracking
DateTime?startTime = CurrentTime;
var lastFrameTime = CurrentTime;
// This will be false when the image is not available
while (reader.isOpened())
{
if (!thread_running)
{
break;
}
// Setup recording
RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false,
Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs,
RecordGaze, RecordHOG, RecordTracked, RecordAligned, true,
reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0);
RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root);
// Detect faces here and return bounding boxes
List <Rect> face_detections = new List <Rect>();
List <float> confidences = new List <float>();
if (DetectorHOG)
{
face_detector.DetectFacesHOG(face_detections, gray_frame, confidences);
}
else if (DetectorCNN)
{
face_detector.DetectFacesMTCNN(face_detections, frame, confidences);
}
else if (DetectorHaar)
{
face_detector.DetectFacesHaar(face_detections, gray_frame, confidences);
}
// For visualization
double progress = reader.GetProgress();
for (int i = 0; i < face_detections.Count; ++i)
{
bool detection_succeeding = landmark_detector.DetectFaceLandmarksInImage(frame, face_detections[i], face_model_params, gray_frame);
var landmarks = landmark_detector.CalculateAllLandmarks();
// Predict action units
var au_preds = face_analyser.PredictStaticAUsAndComputeFeatures(frame, landmarks);
// Predic eye gaze
gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
// Only the final face will contain the details
VisualizeFeatures(frame, visualizer_of, landmarks, landmark_detector.GetVisibilities(), detection_succeeding, i == 0, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress);
// Record an observation
RecordObservation(recorder, visualizer_of.GetVisImage(), i, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0, 0);
}
frame = new RawImage(reader.GetNextImage());
gray_frame = new RawImage(reader.GetCurrentFrameGray());
// Write out the tracked image
if (RecordTracked)
{
recorder.WriteObservationTracked();
}
// Do not cary state accross images
landmark_detector.Reset();
face_analyser.Reset();
recorder.Close();
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
// TODO how to report errors from the reader here? exceptions? logging? Problem for future versions?
}
EndMode();
}
// Capturing and processing the video frame by frame
private void VideoLoop(UtilitiesOF.SequenceReader reader)
{
Thread.CurrentThread.IsBackground = true;
String root = AppDomain.CurrentDomain.BaseDirectory;
FaceModelParameters model_params = new FaceModelParameters(root, true, false, false);
// Initialize the face detector
FaceDetector face_detector = new FaceDetector(model_params.GetHaarLocation(), model_params.GetMTCNNLocation());
// If MTCNN model not available, use HOG
if (!face_detector.IsMTCNNLoaded())
{
model_params.SetFaceDetector(false, true, false);
}
CLNF face_model = new CLNF(model_params);
GazeAnalyserManaged gaze_analyser = new GazeAnalyserManaged();
DateTime?startTime = CurrentTime;
var lastFrameTime = CurrentTime;
while (running)
{
//////////////////////////////////////////////
// CAPTURE FRAME AND DETECT LANDMARKS FOLLOWED BY THE REQUIRED IMAGE PROCESSING
//////////////////////////////////////////////
RawImage frame = reader.GetNextImage();
lastFrameTime = CurrentTime;
processing_fps.AddFrame();
var grayFrame = reader.GetCurrentFrameGray();
if (mirror_image)
{
frame.Mirror();
grayFrame.Mirror();
}
bool detectionSucceeding = ProcessFrame(face_model, gaze_analyser, model_params, frame, grayFrame, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
lock (recording_lock)
{
if (recording)
{
// Add objects to recording queues
List <float> pose = new List <float>();
face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
RawImage image = new RawImage(frame);
recording_objects.Enqueue(new Tuple <RawImage, bool, List <float> >(image, detectionSucceeding, pose));
}
}
List <Tuple <System.Windows.Point, System.Windows.Point> > lines = null;
List <Tuple <float, float> > eye_landmarks = null;
List <System.Windows.Point> landmarks = new List <System.Windows.Point>();
List <Tuple <System.Windows.Point, System.Windows.Point> > gaze_lines = null;
Tuple <float, float> gaze_angle = new Tuple <float, float>(0, 0);
var visibilities = face_model.GetVisibilities();
double scale = face_model.GetRigidParams()[0];
if (detectionSucceeding)
{
List <Tuple <float, float> > landmarks_doubles = face_model.CalculateAllLandmarks();
foreach (var p in landmarks_doubles)
{
landmarks.Add(new System.Windows.Point(p.Item1, p.Item2));
}
eye_landmarks = face_model.CalculateVisibleEyeLandmarks();
gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
gaze_angle = gaze_analyser.GetGazeAngle();
lines = face_model.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
}
if (reset)
{
face_model.Reset();
reset = false;
}
// Visualisation updating
try
{
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
if (latest_img == null)
{
latest_img = frame.CreateWriteableBitmap();
}
List <float> pose = new List <float>();
face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
int yaw = (int)(pose[4] * 180 / Math.PI + 0.5);
int yaw_abs = Math.Abs(yaw);
int roll = (int)(pose[5] * 180 / Math.PI + 0.5);
int roll_abs = Math.Abs(roll);
int pitch = (int)(pose[3] * 180 / Math.PI + 0.5);
int pitch_abs = Math.Abs(pitch);
YawLabel.Content = yaw_abs + "°";
RollLabel.Content = roll_abs + "°";
PitchLabel.Content = pitch_abs + "°";
if (yaw > 0)
{
YawLabelDir.Content = "Right";
}
else if (yaw < 0)
{
YawLabelDir.Content = "Left";
}
else
{
YawLabelDir.Content = "Straight";
}
if (pitch > 0)
{
PitchLabelDir.Content = "Down";
}
else if (pitch < 0)
{
PitchLabelDir.Content = "Up";
}
else
{
PitchLabelDir.Content = "Straight";
}
if (roll > 0)
{
RollLabelDir.Content = "Left";
}
else if (roll < 0)
{
RollLabelDir.Content = "Right";
}
else
{
RollLabelDir.Content = "Straight";
}
XPoseLabel.Content = (int)pose[0] + " mm";
YPoseLabel.Content = (int)pose[1] + " mm";
ZPoseLabel.Content = (int)pose[2] + " mm";
String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI));
String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI));
YawLabelGaze.Content = x_angle;
PitchLabelGaze.Content = y_angle;
if (gaze_angle.Item1 > 0)
{
YawLabelGazeDir.Content = "Right";
}
else if (gaze_angle.Item1 < 0)
{
YawLabelGazeDir.Content = "Left";
}
else
{
YawLabelGazeDir.Content = "Straight";
}
if (gaze_angle.Item2 > 0)
{
PitchLabelGazeDir.Content = "Down";
}
else if (gaze_angle.Item2 < 0)
{
PitchLabelGazeDir.Content = "Up";
}
else
{
PitchLabelGazeDir.Content = "Straight";
}
double confidence = face_model.GetConfidence();
if (confidence < 0)
{
confidence = 0;
}
else if (confidence > 1)
{
confidence = 1;
}
frame.UpdateWriteableBitmap(latest_img);
webcam_img.Clear();
webcam_img.Source = latest_img;
webcam_img.Confidence.Add(confidence);
webcam_img.FPS = processing_fps.GetFPS();
if (detectionSucceeding)
{
webcam_img.OverlayLines.Add(lines);
webcam_img.OverlayPoints.Add(landmarks);
webcam_img.OverlayPointsVisibility.Add(visibilities);
webcam_img.FaceScale.Add(scale);
List <System.Windows.Point> eye_landmark_points = new List <System.Windows.Point>();
foreach (var p in eye_landmarks)
{
eye_landmark_points.Add(new System.Windows.Point(p.Item1, p.Item2));
}
webcam_img.OverlayEyePoints.Add(eye_landmark_points);
webcam_img.GazeLines.Add(gaze_lines);
// Publish the information for other applications
String str_head_pose = String.Format("{0}:{1:F2}, {2:F2}, {3:F2}, {4:F2}, {5:F2}, {6:F2}", "HeadPose", pose[0], pose[1], pose[2],
pose[3] * 180 / Math.PI, pose[4] * 180 / Math.PI, pose[5] * 180 / Math.PI);
zero_mq_socket.Send(new ZFrame(str_head_pose, Encoding.UTF8));
String str_gaze = String.Format("{0}:{1:F2}, {2:F2}", "GazeAngle", gaze_angle.Item1 * (180.0 / Math.PI), gaze_angle.Item2 * (180.0 / Math.PI));
zero_mq_socket.Send(new ZFrame(str_gaze, Encoding.UTF8));
}
}));
while (running & pause)
{
Thread.Sleep(10);
}
}
catch (TaskCanceledException)
{
// Quitting
break;
}
}
reader.Close();
System.Console.Out.WriteLine("Thread finished");
}
public MainWindow()
{
InitializeComponent();
// Set the icon
Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);
this.Icon = BitmapFrame.Create(iconUri);
String root = AppDomain.CurrentDomain.BaseDirectory;
// TODO, create a demo version of parameters
face_model_params = new FaceModelParameters(root, true, false, false);
face_model_params.optimiseForVideo();
// Initialize the face detector
FaceDetector face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());
// If MTCNN model not available, use HOG
if (!face_detector.IsMTCNNLoaded())
{
face_model_params.SetFaceDetector(false, true, false);
}
landmark_detector = new CLNF(face_model_params);
face_analyser = new FaceAnalyserManaged(root, true, 112, true);
gaze_analyser = new GazeAnalyserManaged();
Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
{
headPosePlot.AssocColor(0, Colors.Blue);
headPosePlot.AssocColor(1, Colors.Red);
headPosePlot.AssocColor(2, Colors.Green);
headPosePlot.AssocName(1, "Turn");
headPosePlot.AssocName(2, "Tilt");
headPosePlot.AssocName(0, "Up/Down");
headPosePlot.AssocThickness(0, 2);
headPosePlot.AssocThickness(1, 2);
headPosePlot.AssocThickness(2, 2);
gazePlot.AssocColor(0, Colors.Red);
gazePlot.AssocColor(1, Colors.Blue);
gazePlot.AssocName(0, "Left-right");
gazePlot.AssocName(1, "Up-down");
gazePlot.AssocThickness(0, 2);
gazePlot.AssocThickness(1, 2);
smilePlot.AssocColor(0, Colors.Green);
smilePlot.AssocColor(1, Colors.Red);
smilePlot.AssocName(0, "Smile");
smilePlot.AssocName(1, "Frown");
smilePlot.AssocThickness(0, 2);
smilePlot.AssocThickness(1, 2);
browPlot.AssocColor(0, Colors.Green);
browPlot.AssocColor(1, Colors.Red);
browPlot.AssocName(0, "Raise");
browPlot.AssocName(1, "Furrow");
browPlot.AssocThickness(0, 2);
browPlot.AssocThickness(1, 2);
eyePlot.AssocColor(0, Colors.Green);
eyePlot.AssocColor(1, Colors.Red);
eyePlot.AssocName(0, "Eye widen");
eyePlot.AssocName(1, "Nose wrinkle");
eyePlot.AssocThickness(0, 2);
eyePlot.AssocThickness(1, 2);
}));
}
