コード例 #1
0
        private void RecordObservation(RecorderOpenFace recorder, RawImage vis_image, int face_id, bool success, float fx, float fy, float cx, float cy, double timestamp, int frame_number)
        {
            recorder.SetObservationTimestamp(timestamp);

            double confidence = landmark_detector.GetConfidence();

            List <float> pose = new List <float>();

            landmark_detector.GetPose(pose, fx, fy, cx, cy);
            recorder.SetObservationPose(pose);

            List <Tuple <float, float> >        landmarks_2D = landmark_detector.CalculateAllLandmarks();
            List <Tuple <float, float, float> > landmarks_3D = landmark_detector.Calculate3DLandmarks(fx, fy, cx, cy);
            List <float> global_params = landmark_detector.GetRigidParams();
            List <float> local_params  = landmark_detector.GetNonRigidParams();

            recorder.SetObservationLandmarks(landmarks_2D, landmarks_3D, global_params, local_params, confidence, success);

            var gaze       = gaze_analyser.GetGazeCamera();
            var gaze_angle = gaze_analyser.GetGazeAngle();

            var landmarks_2d_eyes = landmark_detector.CalculateAllEyeLandmarks();
            var landmarks_3d_eyes = landmark_detector.CalculateAllEyeLandmarks3D(fx, fy, cx, cy);

            recorder.SetObservationGaze(gaze.Item1, gaze.Item2, gaze_angle, landmarks_2d_eyes, landmarks_3d_eyes);

            var au_regs    = face_analyser.GetCurrentAUsReg();
            var au_classes = face_analyser.GetCurrentAUsClass();

            recorder.SetObservationActionUnits(au_regs, au_classes);

            recorder.SetObservationFaceID(face_id);
            recorder.SetObservationFrameNumber(frame_number);

            recorder.SetObservationFaceAlign(face_analyser.GetLatestAlignedFace());

            var hog_feature = face_analyser.GetLatestHOGFeature();

            recorder.SetObservationHOG(success, hog_feature, face_analyser.GetHOGRows(), face_analyser.GetHOGCols(), face_analyser.GetHOGChannels());

            recorder.SetObservationVisualization(vis_image);

            recorder.WriteObservation();
        }
コード例 #2
0
        // The main function call for processing the webcam feed
        private void ProcessingLoop(SequenceReader reader)
        {
            thread_running = true;

            Thread.CurrentThread.IsBackground = true;

            DateTime?startTime = CurrentTime;

            var lastFrameTime = CurrentTime;

            landmark_detector.Reset();
            face_analyser.Reset();

            int frame_id = 0;

            double old_gaze_x = 0;
            double old_gaze_y = 0;

            double smile_cumm     = 0;
            double frown_cumm     = 0;
            double brow_up_cumm   = 0;
            double brow_down_cumm = 0;
            double widen_cumm     = 0;
            double wrinkle_cumm   = 0;

            while (thread_running)
            {
                // Loading an image file
                RawImage frame      = new RawImage(reader.GetNextImage());
                RawImage gray_frame = new RawImage(reader.GetCurrentFrameGray());

                lastFrameTime = CurrentTime;
                processing_fps.AddFrame();

                bool detection_succeeding = landmark_detector.DetectLandmarksInVideo(gray_frame, face_model_params);

                // The face analysis step (only done if recording AUs, HOGs or video)
                face_analyser.AddNextFrame(frame, landmark_detector.CalculateAllLandmarks(), detection_succeeding, true);
                gaze_analyser.AddNextFrame(landmark_detector, detection_succeeding, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());

                double confidence = landmark_detector.GetConfidence();

                if (confidence < 0)
                {
                    confidence = 0;
                }
                else if (confidence > 1)
                {
                    confidence = 1;
                }

                List <double> pose = new List <double>();

                landmark_detector.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());

                List <double> non_rigid_params = landmark_detector.GetNonRigidParams();
                double        scale            = landmark_detector.GetRigidParams()[0];

                double time_stamp = (DateTime.Now - (DateTime)startTime).TotalMilliseconds;


                List <Tuple <Point, Point> >   lines         = null;
                List <Tuple <double, double> > landmarks     = null;
                List <Tuple <double, double> > eye_landmarks = null;
                List <Tuple <Point, Point> >   gaze_lines    = null;
                Tuple <double, double>         gaze_angle    = gaze_analyser.GetGazeAngle();

                if (detection_succeeding)
                {
                    landmarks     = landmark_detector.CalculateVisibleLandmarks();
                    eye_landmarks = landmark_detector.CalculateVisibleEyeLandmarks();
                    lines         = landmark_detector.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
                    gaze_lines    = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
                }

                // Visualisation
                Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
                {
                    var au_regs = face_analyser.GetCurrentAUsReg();
                    if (au_regs.Count > 0)
                    {
                        double smile = (au_regs["AU12"] + au_regs["AU06"] + au_regs["AU25"]) / 13.0;
                        double frown = (au_regs["AU15"] + au_regs["AU17"]) / 12.0;

                        double brow_up = (au_regs["AU01"] + au_regs["AU02"]) / 10.0;
                        double brow_down = au_regs["AU04"] / 5.0;

                        double eye_widen = au_regs["AU05"] / 3.0;
                        double nose_wrinkle = au_regs["AU09"] / 4.0;

                        Dictionary <int, double> smileDict = new Dictionary <int, double>();
                        smileDict[0] = 0.7 * smile_cumm + 0.3 * smile;
                        smileDict[1] = 0.7 * frown_cumm + 0.3 * frown;
                        smilePlot.AddDataPoint(new DataPointGraph()
                        {
                            Time = CurrentTime, values = smileDict, Confidence = confidence
                        });

                        Dictionary <int, double> browDict = new Dictionary <int, double>();
                        browDict[0] = 0.7 * brow_up_cumm + 0.3 * brow_up;
                        browDict[1] = 0.7 * brow_down_cumm + 0.3 * brow_down;
                        browPlot.AddDataPoint(new DataPointGraph()
                        {
                            Time = CurrentTime, values = browDict, Confidence = confidence
                        });

                        Dictionary <int, double> eyeDict = new Dictionary <int, double>();
                        eyeDict[0] = 0.7 * widen_cumm + 0.3 * eye_widen;
                        eyeDict[1] = 0.7 * wrinkle_cumm + 0.3 * nose_wrinkle;
                        eyePlot.AddDataPoint(new DataPointGraph()
                        {
                            Time = CurrentTime, values = eyeDict, Confidence = confidence
                        });

                        smile_cumm = smileDict[0];
                        frown_cumm = smileDict[1];
                        brow_up_cumm = browDict[0];
                        brow_down_cumm = browDict[1];
                        widen_cumm = eyeDict[0];
                        wrinkle_cumm = eyeDict[1];
                    }
                    else
                    {
                        // If no AUs present disable the AU visualization
                        MainGrid.ColumnDefinitions[2].Width = new GridLength(0);
                        eyePlot.Visibility = Visibility.Collapsed;
                        browPlot.Visibility = Visibility.Collapsed;
                        smilePlot.Visibility = Visibility.Collapsed;
                    }

                    Dictionary <int, double> poseDict = new Dictionary <int, double>();
                    poseDict[0] = -pose[3];
                    poseDict[1] = pose[4];
                    poseDict[2] = pose[5];
                    headPosePlot.AddDataPoint(new DataPointGraph()
                    {
                        Time = CurrentTime, values = poseDict, Confidence = confidence
                    });

                    Dictionary <int, double> gazeDict = new Dictionary <int, double>();
                    gazeDict[0] = gaze_angle.Item1 * (180.0 / Math.PI);
                    gazeDict[0] = 0.5 * old_gaze_x + 0.5 * gazeDict[0];
                    gazeDict[1] = -gaze_angle.Item2 * (180.0 / Math.PI);
                    gazeDict[1] = 0.5 * old_gaze_y + 0.5 * gazeDict[1];
                    gazePlot.AddDataPoint(new DataPointGraph()
                    {
                        Time = CurrentTime, values = gazeDict, Confidence = confidence
                    });

                    old_gaze_x = gazeDict[0];
                    old_gaze_y = gazeDict[1];

                    if (latest_img == null)
                    {
                        latest_img = frame.CreateWriteableBitmap();
                    }

                    frame.UpdateWriteableBitmap(latest_img);

                    video.Source = latest_img;
                    video.Confidence = confidence;
                    video.FPS = processing_fps.GetFPS();

                    if (!detection_succeeding)
                    {
                        video.OverlayLines.Clear();
                        video.OverlayPoints.Clear();
                        video.OverlayEyePoints.Clear();
                        video.GazeLines.Clear();
                    }
                    else
                    {
                        video.OverlayLines = lines;

                        List <Point> landmark_points = new List <Point>();
                        foreach (var p in landmarks)
                        {
                            landmark_points.Add(new Point(p.Item1, p.Item2));
                        }

                        List <Point> eye_landmark_points = new List <Point>();
                        foreach (var p in eye_landmarks)
                        {
                            eye_landmark_points.Add(new Point(p.Item1, p.Item2));
                        }


                        video.OverlayPoints = landmark_points;
                        video.OverlayEyePoints = eye_landmark_points;
                        video.GazeLines = gaze_lines;
                    }
                }));

                if (reset)
                {
                    if (resetPoint.HasValue)
                    {
                        landmark_detector.Reset(resetPoint.Value.X, resetPoint.Value.Y);
                        resetPoint = null;
                    }
                    else
                    {
                        landmark_detector.Reset();
                    }

                    face_analyser.Reset();
                    reset = false;

                    Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
                    {
                        headPosePlot.ClearDataPoints();
                        headPosePlot.ClearDataPoints();
                        gazePlot.ClearDataPoints();
                        smilePlot.ClearDataPoints();
                        browPlot.ClearDataPoints();
                        eyePlot.ClearDataPoints();
                    }));
                }

                frame_id++;
            }
            reader.Close();
            latest_img = null;
        }