public MainWindow()
        {
            InitializeComponent();
            this.DataContext = this; // For WPF data binding

            // Set the icon
            Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);

            this.Icon = BitmapFrame.Create(iconUri);

            String root = AppDomain.CurrentDomain.BaseDirectory;

            face_model_params = new FaceModelParameters(root, LandmarkDetectorCECLM, LandmarkDetectorCLNF, LandmarkDetectorCLM);
            // Initialize the face detector
            face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());

            // If MTCNN model not available, use HOG
            if (!face_detector.IsMTCNNLoaded())
            {
                FaceDetCNN.IsEnabled = false;
                DetectorCNN          = false;
                DetectorHOG          = true;
            }
            face_model_params.SetFaceDetector(DetectorHaar, DetectorHOG, DetectorCNN);

            landmark_detector = new CLNF(face_model_params);

            gaze_analyser = new GazeAnalyserManaged();
        }
Exemple #2
0
        private void Initialize(object sender, PipelineRunEventArgs e)
        {
            string rootDirectory = AppDomain.CurrentDomain.BaseDirectory;

            faceModelParameters = new FaceModelParameters(rootDirectory, true, false, false);
            faceModelParameters.optimiseForVideo();

            faceDetector = new FaceDetector(faceModelParameters.GetHaarLocation(), faceModelParameters.GetMTCNNLocation());
            if (!faceDetector.IsMTCNNLoaded())
            {
                faceModelParameters.SetFaceDetector(false, true, false);
            }

            landmarkDetector = new CLNF(faceModelParameters);
            faceAnalyser     = new FaceAnalyser(rootDirectory, dynamic: true, output_width: 112, mask_aligned: true);
            gazeAnalyser     = new GazeAnalyser();

            landmarkDetector.Reset();
            faceAnalyser.Reset();
        }
        protected ExtractorBase(FaceModelParameters faceModelParameters)
        {
            if (_initialized)
            {
                return;
            }

            ModelParams  = faceModelParameters;
            GazeAnalyzer = new GazeAnalyserManaged();

            var face_detector = new FaceDetector(ModelParams.GetHaarLocation(), ModelParams.GetMTCNNLocation());

            if (!face_detector.IsMTCNNLoaded()) // If MTCNN model not available, use HOG
            {
                ModelParams.SetFaceDetector(false, true, false);
            }

            FaceModel    = new CLNF(ModelParams);
            _initialized = true;
        }
Exemple #4
0
        // Capturing and processing the video frame by frame
        private void VideoLoop(UtilitiesOF.SequenceReader reader)
        {
            Thread.CurrentThread.IsBackground = true;

            String root = AppDomain.CurrentDomain.BaseDirectory;
            FaceModelParameters model_params = new FaceModelParameters(root, true, false, false);

            // Initialize the face detector
            FaceDetector face_detector = new FaceDetector(model_params.GetHaarLocation(), model_params.GetMTCNNLocation());

            // If MTCNN model not available, use HOG
            if (!face_detector.IsMTCNNLoaded())
            {
                model_params.SetFaceDetector(false, true, false);
            }

            CLNF face_model = new CLNF(model_params);
            GazeAnalyserManaged gaze_analyser = new GazeAnalyserManaged();

            DateTime?startTime = CurrentTime;

            var lastFrameTime = CurrentTime;

            while (running)
            {
                //////////////////////////////////////////////
                // CAPTURE FRAME AND DETECT LANDMARKS FOLLOWED BY THE REQUIRED IMAGE PROCESSING
                //////////////////////////////////////////////

                RawImage frame = reader.GetNextImage();

                lastFrameTime = CurrentTime;
                processing_fps.AddFrame();

                var grayFrame = reader.GetCurrentFrameGray();

                if (mirror_image)
                {
                    frame.Mirror();
                    grayFrame.Mirror();
                }

                bool detectionSucceeding = ProcessFrame(face_model, gaze_analyser, model_params, frame, grayFrame, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());

                lock (recording_lock)
                {
                    if (recording)
                    {
                        // Add objects to recording queues
                        List <float> pose = new List <float>();
                        face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
                        RawImage image = new RawImage(frame);
                        recording_objects.Enqueue(new Tuple <RawImage, bool, List <float> >(image, detectionSucceeding, pose));
                    }
                }

                List <Tuple <System.Windows.Point, System.Windows.Point> > lines = null;
                List <Tuple <float, float> > eye_landmarks = null;
                List <System.Windows.Point>  landmarks     = new List <System.Windows.Point>();
                List <Tuple <System.Windows.Point, System.Windows.Point> > gaze_lines = null;
                Tuple <float, float> gaze_angle = new Tuple <float, float>(0, 0);
                var    visibilities             = face_model.GetVisibilities();
                double scale = face_model.GetRigidParams()[0];

                if (detectionSucceeding)
                {
                    List <Tuple <float, float> > landmarks_doubles = face_model.CalculateAllLandmarks();

                    foreach (var p in landmarks_doubles)
                    {
                        landmarks.Add(new System.Windows.Point(p.Item1, p.Item2));
                    }

                    eye_landmarks = face_model.CalculateVisibleEyeLandmarks();

                    gaze_lines = gaze_analyser.CalculateGazeLines(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
                    gaze_angle = gaze_analyser.GetGazeAngle();

                    lines = face_model.CalculateBox(reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());
                }

                if (reset)
                {
                    face_model.Reset();
                    reset = false;
                }

                // Visualisation updating
                try
                {
                    Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
                    {
                        if (latest_img == null)
                        {
                            latest_img = frame.CreateWriteableBitmap();
                        }

                        List <float> pose = new List <float>();
                        face_model.GetPose(pose, reader.GetFx(), reader.GetFy(), reader.GetCx(), reader.GetCy());

                        int yaw = (int)(pose[4] * 180 / Math.PI + 0.5);
                        int yaw_abs = Math.Abs(yaw);

                        int roll = (int)(pose[5] * 180 / Math.PI + 0.5);
                        int roll_abs = Math.Abs(roll);

                        int pitch = (int)(pose[3] * 180 / Math.PI + 0.5);
                        int pitch_abs = Math.Abs(pitch);

                        YawLabel.Content = yaw_abs + "°";
                        RollLabel.Content = roll_abs + "°";
                        PitchLabel.Content = pitch_abs + "°";

                        if (yaw > 0)
                        {
                            YawLabelDir.Content = "Right";
                        }
                        else if (yaw < 0)
                        {
                            YawLabelDir.Content = "Left";
                        }
                        else
                        {
                            YawLabelDir.Content = "Straight";
                        }

                        if (pitch > 0)
                        {
                            PitchLabelDir.Content = "Down";
                        }
                        else if (pitch < 0)
                        {
                            PitchLabelDir.Content = "Up";
                        }
                        else
                        {
                            PitchLabelDir.Content = "Straight";
                        }

                        if (roll > 0)
                        {
                            RollLabelDir.Content = "Left";
                        }
                        else if (roll < 0)
                        {
                            RollLabelDir.Content = "Right";
                        }
                        else
                        {
                            RollLabelDir.Content = "Straight";
                        }

                        XPoseLabel.Content = (int)pose[0] + " mm";
                        YPoseLabel.Content = (int)pose[1] + " mm";
                        ZPoseLabel.Content = (int)pose[2] + " mm";

                        String x_angle = String.Format("{0:F0}°", gaze_angle.Item1 * (180.0 / Math.PI));
                        String y_angle = String.Format("{0:F0}°", gaze_angle.Item2 * (180.0 / Math.PI));
                        YawLabelGaze.Content = x_angle;
                        PitchLabelGaze.Content = y_angle;

                        if (gaze_angle.Item1 > 0)
                        {
                            YawLabelGazeDir.Content = "Right";
                        }
                        else if (gaze_angle.Item1 < 0)
                        {
                            YawLabelGazeDir.Content = "Left";
                        }
                        else
                        {
                            YawLabelGazeDir.Content = "Straight";
                        }

                        if (gaze_angle.Item2 > 0)
                        {
                            PitchLabelGazeDir.Content = "Down";
                        }
                        else if (gaze_angle.Item2 < 0)
                        {
                            PitchLabelGazeDir.Content = "Up";
                        }
                        else
                        {
                            PitchLabelGazeDir.Content = "Straight";
                        }

                        double confidence = face_model.GetConfidence();

                        if (confidence < 0)
                        {
                            confidence = 0;
                        }
                        else if (confidence > 1)
                        {
                            confidence = 1;
                        }

                        frame.UpdateWriteableBitmap(latest_img);
                        webcam_img.Clear();

                        webcam_img.Source = latest_img;
                        webcam_img.Confidence.Add(confidence);
                        webcam_img.FPS = processing_fps.GetFPS();
                        if (detectionSucceeding)
                        {
                            webcam_img.OverlayLines.Add(lines);
                            webcam_img.OverlayPoints.Add(landmarks);
                            webcam_img.OverlayPointsVisibility.Add(visibilities);
                            webcam_img.FaceScale.Add(scale);

                            List <System.Windows.Point> eye_landmark_points = new List <System.Windows.Point>();
                            foreach (var p in eye_landmarks)
                            {
                                eye_landmark_points.Add(new System.Windows.Point(p.Item1, p.Item2));
                            }


                            webcam_img.OverlayEyePoints.Add(eye_landmark_points);
                            webcam_img.GazeLines.Add(gaze_lines);

                            // Publish the information for other applications
                            String str_head_pose = String.Format("{0}:{1:F2}, {2:F2}, {3:F2}, {4:F2}, {5:F2}, {6:F2}", "HeadPose", pose[0], pose[1], pose[2],
                                                                 pose[3] * 180 / Math.PI, pose[4] * 180 / Math.PI, pose[5] * 180 / Math.PI);

                            zero_mq_socket.Send(new ZFrame(str_head_pose, Encoding.UTF8));

                            String str_gaze = String.Format("{0}:{1:F2}, {2:F2}", "GazeAngle", gaze_angle.Item1 * (180.0 / Math.PI), gaze_angle.Item2 * (180.0 / Math.PI));

                            zero_mq_socket.Send(new ZFrame(str_gaze, Encoding.UTF8));
                        }
                    }));

                    while (running & pause)
                    {
                        Thread.Sleep(10);
                    }
                }
                catch (TaskCanceledException)
                {
                    // Quitting
                    break;
                }
            }
            reader.Close();
            System.Console.Out.WriteLine("Thread finished");
        }
Exemple #5
0
        public MainWindow()
        {
            InitializeComponent();

            // Set the icon
            Uri iconUri = new Uri("logo1.ico", UriKind.RelativeOrAbsolute);

            this.Icon = BitmapFrame.Create(iconUri);

            String root = AppDomain.CurrentDomain.BaseDirectory;

            // TODO, create a demo version of parameters
            face_model_params = new FaceModelParameters(root, true, false, false);
            face_model_params.optimiseForVideo();

            // Initialize the face detector
            FaceDetector face_detector = new FaceDetector(face_model_params.GetHaarLocation(), face_model_params.GetMTCNNLocation());

            // If MTCNN model not available, use HOG
            if (!face_detector.IsMTCNNLoaded())
            {
                face_model_params.SetFaceDetector(false, true, false);
            }

            landmark_detector = new CLNF(face_model_params);
            face_analyser     = new FaceAnalyserManaged(root, true, 112, true);
            gaze_analyser     = new GazeAnalyserManaged();

            Dispatcher.Invoke(DispatcherPriority.Render, new TimeSpan(0, 0, 0, 0, 200), (Action)(() =>
            {
                headPosePlot.AssocColor(0, Colors.Blue);
                headPosePlot.AssocColor(1, Colors.Red);
                headPosePlot.AssocColor(2, Colors.Green);

                headPosePlot.AssocName(1, "Turn");
                headPosePlot.AssocName(2, "Tilt");
                headPosePlot.AssocName(0, "Up/Down");

                headPosePlot.AssocThickness(0, 2);
                headPosePlot.AssocThickness(1, 2);
                headPosePlot.AssocThickness(2, 2);

                gazePlot.AssocColor(0, Colors.Red);
                gazePlot.AssocColor(1, Colors.Blue);

                gazePlot.AssocName(0, "Left-right");
                gazePlot.AssocName(1, "Up-down");
                gazePlot.AssocThickness(0, 2);
                gazePlot.AssocThickness(1, 2);

                smilePlot.AssocColor(0, Colors.Green);
                smilePlot.AssocColor(1, Colors.Red);
                smilePlot.AssocName(0, "Smile");
                smilePlot.AssocName(1, "Frown");
                smilePlot.AssocThickness(0, 2);
                smilePlot.AssocThickness(1, 2);

                browPlot.AssocColor(0, Colors.Green);
                browPlot.AssocColor(1, Colors.Red);
                browPlot.AssocName(0, "Raise");
                browPlot.AssocName(1, "Furrow");
                browPlot.AssocThickness(0, 2);
                browPlot.AssocThickness(1, 2);

                eyePlot.AssocColor(0, Colors.Green);
                eyePlot.AssocColor(1, Colors.Red);
                eyePlot.AssocName(0, "Eye widen");
                eyePlot.AssocName(1, "Nose wrinkle");
                eyePlot.AssocThickness(0, 2);
                eyePlot.AssocThickness(1, 2);
            }));
        }