private void RecordObservation(RecorderOpenFace recorder, RawImage vis_image, int face_id, bool success, float fx, float fy, float cx, float cy, double timestamp, int frame_number) { recorder.SetObservationTimestamp(timestamp); double confidence = landmark_detector.GetConfidence(); List <float> pose = new List <float>(); landmark_detector.GetPose(pose, fx, fy, cx, cy); recorder.SetObservationPose(pose); List <Tuple <float, float> > landmarks_2D = landmark_detector.CalculateAllLandmarks(); List <Tuple <float, float, float> > landmarks_3D = landmark_detector.Calculate3DLandmarks(fx, fy, cx, cy); List <float> global_params = landmark_detector.GetRigidParams(); List <float> local_params = landmark_detector.GetNonRigidParams(); recorder.SetObservationLandmarks(landmarks_2D, landmarks_3D, global_params, local_params, confidence, success); var gaze = gaze_analyser.GetGazeCamera(); var gaze_angle = gaze_analyser.GetGazeAngle(); var landmarks_2d_eyes = landmark_detector.CalculateAllEyeLandmarks(); var landmarks_3d_eyes = landmark_detector.CalculateAllEyeLandmarks3D(fx, fy, cx, cy); recorder.SetObservationGaze(gaze.Item1, gaze.Item2, gaze_angle, landmarks_2d_eyes, landmarks_3d_eyes); var au_regs = face_analyser.GetCurrentAUsReg(); var au_classes = face_analyser.GetCurrentAUsClass(); recorder.SetObservationActionUnits(au_regs, au_classes); recorder.SetObservationFaceID(face_id); recorder.SetObservationFrameNumber(frame_number); recorder.SetObservationFaceAlign(face_analyser.GetLatestAlignedFace()); var hog_feature = face_analyser.GetLatestHOGFeature(); recorder.SetObservationHOG(success, hog_feature, face_analyser.GetHOGRows(), face_analyser.GetHOGCols(), face_analyser.GetHOGChannels()); recorder.SetObservationVisualization(vis_image); recorder.WriteObservation(); }
private void ProcessIndividualImages(ImageReader reader) { // Make sure the GUI is setup appropriately SetupFeatureExtractionMode(); // Indicate we will start running the thread thread_running = true; // Reload the face landmark detector if needed ReloadLandmarkDetector(); if (!landmark_detector.isLoaded()) { DetectorNotFoundWarning(); EndMode(); thread_running = false; return; } // Setup the parameters optimized for working on individual images rather than sequences face_model_params.optimiseForImages(); // Setup the visualization Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance, false); // Initialize the face detector if it has not been initialized yet if (face_detector == null) { face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation()); } // Initialize the face analyser face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, false, image_output_size, MaskAligned); // Loading an image file var frame = new RawImage(reader.GetNextImage()); var gray_frame = new RawImage(reader.GetCurrentFrameGray()); // For FPS tracking DateTime?startTime = CurrentTime; var lastFrameTime = CurrentTime; // This will be false when the image is not available while (reader.isOpened()) { if (!thread_running) { break; } // Setup recording RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, RecordGaze, RecordHOG, RecordTracked, RecordAligned, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0); RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); // Detect faces here and return bounding boxes List <Rect> face_detections = new List <Rect>(); List <float> confidences = new List <float>(); if (DetectorHOG) { face_detector.DetectFacesHOG(face_detections, gray_frame, confidences); } else if (DetectorCNN) { face_detector.DetectFacesMTCNN(face_detections, frame, confidences); } else if (DetectorHaar) { face_detector.DetectFacesHaar(face_detections, gray_frame, confidences); } // For visualization double progress = reader.GetProgress(); for (int i = 0; i < face_detections.Count; ++i) { bool detection_succeeding = landmark_detector.DetectFaceLandmarksInImage(frame, face_detections[i], face_model_params, gray_frame); var landmarks = landmark_detector.CalculateAllLandmarks(); // Predict action units var au_preds = face_analyser.PredictStaticAUsAndComputeFeatures(frame, landmarks); // Predic eye gaze gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details VisualizeFeatures(frame, visualizer_of, landmarks, landmark_detector.GetVisibilities(), detection_succeeding, i == 0, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation RecordObservation(recorder, visualizer_of.GetVisImage(), i, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0, 0); } frame = new RawImage(reader.GetNextImage()); gray_frame = new RawImage(reader.GetCurrentFrameGray()); // Write out the tracked image if (RecordTracked) { recorder.WriteObservationTracked(); } // Do not cary state accross images landmark_detector.Reset(); face_analyser.Reset(); recorder.Close(); lastFrameTime = CurrentTime; processing_fps.AddFrame(); // TODO how to report errors from the reader here? exceptions? logging? Problem for future versions? } EndMode(); }
// The main function call for processing sequences private void ProcessSequence(SequenceReader reader) { Thread.CurrentThread.Priority = ThreadPriority.Highest; SetupFeatureExtractionMode(); thread_running = true; // Reload the face landmark detector if needed ReloadLandmarkDetector(); if (!landmark_detector.isLoaded()) { DetectorNotFoundWarning(); EndMode(); thread_running = false; return; } // Set the face detector face_model_params.SetFaceDetector(DetectorHaar, DetectorHOG, DetectorCNN); face_model_params.optimiseForVideo(); // Setup the visualization Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance, false); // Initialize the face analyser face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, DynamicAUModels, image_output_size, MaskAligned); // Reset the tracker landmark_detector.Reset(); // Loading an image file var frame = new RawImage(reader.GetNextImage()); var gray_frame = new RawImage(reader.GetCurrentFrameGray()); // Setup recording RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(true, reader.IsWebcam(), Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, RecordGaze, RecordHOG, RecordTracked, RecordAligned, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetFPS()); RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); // For FPS tracking DateTime?startTime = CurrentTime; var lastFrameTime = CurrentTime; // Empty image would indicate that the stream is over while (!gray_frame.IsEmpty) { if (!thread_running) { break; } double progress = reader.GetProgress(); bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(frame, face_model_params, gray_frame); // The face analysis step (for AUs and eye gaze) face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, false); gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation RecordObservation(recorder, visualizer_of.GetVisImage(), 0, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp(), reader.GetFrameNumber()); if (RecordTracked) { recorder.WriteObservationTracked(); } while (thread_running & thread_paused && skip_frames == 0) { Thread.Sleep(10); } if (skip_frames > 0) { skip_frames--; } frame = new RawImage(reader.GetNextImage()); gray_frame = new RawImage(reader.GetCurrentFrameGray()); lastFrameTime = CurrentTime; processing_fps.AddFrame(); } // Finalize the recording and flush to disk recorder.Close(); // Post-process the AU recordings if (RecordAUs) { face_analyser.PostProcessOutputFile(recorder.GetCSVFile()); } // Close the open video/webcam reader.Close(); EndMode(); }