// The main function call for processing sequences private void ProcessSequence(SequenceReader reader) { Thread.CurrentThread.Priority = ThreadPriority.Highest; SetupFeatureExtractionMode(); thread_running = true; // Reload the face landmark detector if needed ReloadLandmarkDetector(); if (!landmark_detector.isLoaded()) { DetectorNotFoundWarning(); EndMode(); thread_running = false; return; } // Set the face detector face_model_params.SetFaceDetector(DetectorHaar, DetectorHOG, DetectorCNN); face_model_params.optimiseForVideo(); // Setup the visualization Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance, false); // Initialize the face analyser face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, DynamicAUModels, image_output_size, MaskAligned); // Reset the tracker landmark_detector.Reset(); // Loading an image file var frame = new RawImage(reader.GetNextImage()); var gray_frame = new RawImage(reader.GetCurrentFrameGray()); // Setup recording RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(true, reader.IsWebcam(), Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, RecordGaze, RecordHOG, RecordTracked, RecordAligned, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetFPS()); RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); // For FPS tracking DateTime?startTime = CurrentTime; var lastFrameTime = CurrentTime; // Empty image would indicate that the stream is over while (!gray_frame.IsEmpty) { if (!thread_running) { break; } double progress = reader.GetProgress(); bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(frame, face_model_params, gray_frame); // The face analysis step (for AUs and eye gaze) face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, false); gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details VisualizeFeatures(frame, visualizer_of, landmark_detector.CalculateAllLandmarks(), landmark_detector.GetVisibilities(), detection_succeeding, true, false, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation RecordObservation(recorder, visualizer_of.GetVisImage(), 0, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), reader.GetTimestamp(), reader.GetFrameNumber()); if (RecordTracked) { recorder.WriteObservationTracked(); } while (thread_running & thread_paused && skip_frames == 0) { Thread.Sleep(10); } if (skip_frames > 0) { skip_frames--; } frame = new RawImage(reader.GetNextImage()); gray_frame = new RawImage(reader.GetCurrentFrameGray()); lastFrameTime = CurrentTime; processing_fps.AddFrame(); } // Finalize the recording and flush to disk recorder.Close(); // Post-process the AU recordings if (RecordAUs) { face_analyser.PostProcessOutputFile(recorder.GetCSVFile()); } // Close the open video/webcam reader.Close(); EndMode(); }
private void ProcessIndividualImages(ImageReader reader) { // Make sure the GUI is setup appropriately SetupFeatureExtractionMode(); // Indicate we will start running the thread thread_running = true; // Reload the face landmark detector if needed ReloadLandmarkDetector(); if (!landmark_detector.isLoaded()) { DetectorNotFoundWarning(); EndMode(); thread_running = false; return; } // Setup the parameters optimized for working on individual images rather than sequences face_model_params.optimiseForImages(); // Setup the visualization Visualizer visualizer_of = new Visualizer(ShowTrackedVideo || RecordTracked, ShowAppearance, ShowAppearance, false); // Initialize the face detector if it has not been initialized yet if (face_detector == null) { face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation()); } // Initialize the face analyser face_analyser = new FaceAnalyserManaged(AppDomain.CurrentDomain.BaseDirectory, false, image_output_size, MaskAligned); // Loading an image file var frame = reader.GetNextImage(); var gray_frame = reader.GetCurrentFrameGray(); // For FPS tracking DateTime?startTime = CurrentTime; var lastFrameTime = CurrentTime; // This will be false when the image is not available while (reader.isOpened()) { if (!thread_running) { break; } // Setup recording RecorderOpenFaceParameters rec_params = new RecorderOpenFaceParameters(false, false, Record2DLandmarks, Record3DLandmarks, RecordModelParameters, RecordPose, RecordAUs, RecordGaze, RecordHOG, RecordTracked, RecordAligned, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0); RecorderOpenFace recorder = new RecorderOpenFace(reader.GetName(), rec_params, record_root); visualizer_of.SetImage(frame, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Detect faces here and return bounding boxes List <Rect> face_detections = new List <Rect>(); List <float> confidences = new List <float>(); if (DetectorHOG) { face_detector.DetectFacesHOG(face_detections, gray_frame, confidences); } else if (DetectorCNN) { face_detector.DetectFacesMTCNN(face_detections, frame, confidences); } else if (DetectorHaar) { face_detector.DetectFacesHaar(face_detections, gray_frame, confidences); } // For visualization double progress = reader.GetProgress(); for (int i = 0; i < face_detections.Count; ++i) { bool detection_succeeding = landmark_detector.DetectFaceLandmarksInImage(frame, face_detections[i], face_model_params, gray_frame); var landmarks = landmark_detector.CalculateAllLandmarks(); // Predict action units var au_preds = face_analyser.PredictStaticAUsAndComputeFeatures(frame, landmarks); // Predic eye gaze gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy()); // Only the final face will contain the details VisualizeFeatures(frame, visualizer_of, landmarks, landmark_detector.GetVisibilities(), detection_succeeding, i == 0, true, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), progress); // Record an observation RecordObservation(recorder, visualizer_of.GetVisImage(), i, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy(), 0, 0); } recorder.SetObservationVisualization(visualizer_of.GetVisImage()); frame = reader.GetNextImage(); gray_frame = reader.GetCurrentFrameGray(); // Write out the tracked image if (RecordTracked) { recorder.WriteObservationTracked(); } // Do not cary state accross images landmark_detector.Reset(); face_analyser.Reset(); recorder.Close(); lastFrameTime = CurrentTime; processing_fps.AddFrame(); // TODO how to report errors from the reader here? exceptions? logging? Problem for future versions? } EndMode(); }