// Use this for initialization void Start () { //initialize FaceTracker FaceTracker faceTracker = new FaceTracker (Utils.getFilePath ("tracker_model.json")); //initialize FaceTrackerParams FaceTrackerParams faceTrackerParams = new FaceTrackerParams (); gameObject.transform.localScale = new Vector3 (imgTexture.width, imgTexture.height, 1); Camera.main.orthographicSize = imgTexture.height / 2; Mat imgMat = new Mat (imgTexture.height, imgTexture.width, CvType.CV_8UC4); Utils.texture2DToMat (imgTexture, imgMat); Debug.Log ("imgMat dst ToString " + imgMat.ToString ()); CascadeClassifier cascade = new CascadeClassifier (Utils.getFilePath ("haarcascade_frontalface_alt.xml")); if (cascade.empty ()) { Debug.LogError ("cascade file is not loaded.Please copy from “FaceTrackerSample/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); } //convert image to greyscale Mat gray = new Mat (); Imgproc.cvtColor (imgMat, gray, Imgproc.COLOR_RGBA2GRAY); MatOfRect faces = new MatOfRect (); Imgproc.equalizeHist (gray, gray); cascade.detectMultiScale (gray, faces, 1.1f, 2, 0 // | Objdetect.CASCADE_FIND_BIGGEST_OBJECT | Objdetect.CASCADE_SCALE_IMAGE, new OpenCVForUnity.Size (gray.cols () * 0.05, gray.cols () * 0.05), new Size ()); Debug.Log ("faces " + faces.dump ()); if (faces.rows () > 0) { //add initial face points from MatOfRect faceTracker.addPoints (faces); } //track face points.if face points <= 0, always return false. if (faceTracker.track (imgMat, faceTrackerParams)) faceTracker.draw (imgMat, new Scalar (255, 0, 0, 255), new Scalar (0, 255, 0, 255)); Texture2D texture = new Texture2D (imgMat.cols (), imgMat.rows (), TextureFormat.RGBA32, false); Utils.matToTexture2D (imgMat, texture); gameObject.GetComponent<Renderer> ().material.mainTexture = texture; }
// Use this for initialization void Start () { //initialize FaceTracker faceTracker = new FaceTracker (Utils.getFilePath ("tracker_model.json")); //initialize FaceTrackerParams faceTrackerParams = new FaceTrackerParams (); webCamTextureToMatHelper = gameObject.GetComponent<WebCamTextureToMatHelper> (); webCamTextureToMatHelper.Init (OnWebCamTextureToMatHelperInited, OnWebCamTextureToMatHelperDisposed); }
private FaceMatrix(FaceTracker faceTracker, MediaCapture mediaCapture, int rowsCount, int columnsCount) { _faceTracker = faceTracker; _mediaCapture = mediaCapture; // get properties of the stream, we need them to get width/height for face detection var videoProperties = _mediaCapture.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; _previewFrame = new VideoFrame(BitmapPixelFormat.Nv12, (int)videoProperties.Width, (int)videoProperties.Height); _rowsCount = rowsCount; _columnsCount = columnsCount; }
public async Task Initialize(string cameraName = "LifeCam") { // select the camera var devices = await DeviceInformation.FindAllAsync(DeviceClass.VideoCapture); var device = devices.FirstOrDefault(d => d.Name.ToLowerInvariant().Contains(cameraName.ToLower())) ?? devices.FirstOrDefault(); var settings = new MediaCaptureInitializationSettings() { VideoDeviceId = device.Id }; // initialize the camera mediaCapture = new MediaCapture(); await mediaCapture.InitializeAsync(settings); // select a lower framerate and resolution to reduce USB bandwidth var props = mediaCapture .VideoDeviceController .GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview) .Cast <VideoEncodingProperties>() .First(p => p.FrameRate.Numerator == 10 && p.Height == 720); await mediaCapture.VideoDeviceController.SetMediaStreamPropertiesAsync(MediaStreamType.VideoPreview, props); // start the preview feed (a CaptureElement is required to sync the feed) captureElement = new CaptureElement() { Source = mediaCapture }; await mediaCapture.StartPreviewAsync(); // get the video properties var previewProperties = mediaCapture.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; ImageHeight = (int)previewProperties.Height; ImageWidth = (int)previewProperties.Width; // intialize face tracking faceTracker = await FaceTracker.CreateAsync(); // Get the known persons var persons = await faceClient.GetPersonsAsync(personGroupId); personMap = persons.ToDictionary(p => p.PersonId, p => p.Name); }
/// <summary> /// Updates the face tracking information for this skeleton /// </summary> internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest) { this.skeletonTrackingState = skeletonOfInterest.TrackingState; if (this.skeletonTrackingState != SkeletonTrackingState.Tracked) { // nothing to do with an untracked skeleton. return; } if (this.faceTracker == null) { try { this.faceTracker = new FaceTracker(kinectSensor); } catch (InvalidOperationException) { // During some shutdown scenarios the FaceTracker // is unable to be instantiated. Catch that exception // and don't track a face. Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException"); this.faceTracker = null; } } if (this.faceTracker != null) { FaceTrackFrame frame = this.faceTracker.Track( colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest); this.lastFaceTrackSucceeded = frame.TrackSuccessful; if (this.lastFaceTrackSucceeded) { if (faceTriangles == null) { // only need to get this once. It doesn't change. faceTriangles = frame.GetTriangles(); } this.facePoints = frame.GetProjected3DShape(); } } }
public void Run() { //initialize FaceTracker faceTracker = new FaceTracker(tracker_model_json_filepath); //initialize FaceTrackerParams faceTrackerParams = new FaceTrackerParams(); cascade = new CascadeClassifier(); cascade.load(haarcascade_frontalface_alt_xml_filepath); // if (cascade.empty()) // { // Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); // } #if UNITY_ANDROID && !UNITY_EDITOR // Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2). webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true; #endif webCamTextureToMatHelper.Initialize(); if (SpeechRecognizer.ExistsOnDevice()) { resultText.text = "I am running run"; SpeechRecognizerListener listener = GameObject.FindObjectOfType <SpeechRecognizerListener>(); listener.onAuthorizationStatusFetched.AddListener(OnAuthorizationStatusFetched); listener.onAvailabilityChanged.AddListener(OnAvailabilityChange); listener.onErrorDuringRecording.AddListener(OnError); listener.onErrorOnStartRecording.AddListener(OnError); listener.onFinalResults.AddListener(OnFinalResult); listener.onPartialResults.AddListener(OnPartialResult); listener.onEndOfSpeech.AddListener(OnEndOfSpeech); //startRecordingButton.enabled = false; SpeechRecognizer.RequestAccess(); SpeechRecognizer.StartRecording(true); resultText.text = "Say something :-)"; } else { resultText.text = "Sorry, but this device doesn't support speech recognition"; Debug.Log("Next Command is crossfade from run function"); //GameObject.FindGameObjectWithTag("twohand)").GetComponent<Animator>().CrossFade("V", -1); //startRecordingButton.enabled = false; } }
private async Task StartPreviewAsync() { try { _mediaCapture = new MediaCapture(); await _mediaCapture.InitializeAsync(); _displayRequest.RequestActive(); DisplayInformation.AutoRotationPreferences = DisplayOrientations.Landscape; } catch (UnauthorizedAccessException) { // This will be thrown if the user denied access to the camera in privacy settings //("The app was denied access to the camera"); return; } try { PreviewControl.Source = _mediaCapture; await _mediaCapture.StartPreviewAsync(); _isPreviewing = true; } catch (System.IO.FileLoadException) { _mediaCapture.CaptureDeviceExclusiveControlStatusChanged += _mediaCapture_CaptureDeviceExclusiveControlStatusChanged; } //initialize UI facetracking this.faceTracker = await FaceTracker.CreateAsync(); TimeSpan UIUpdateTimer = TimeSpan.FromMilliseconds(66); // 15 fps this.frameProcessingTimer = Windows.System.Threading.ThreadPoolTimer.CreatePeriodicTimer(new Windows.System.Threading.TimerElapsedHandler(ProcessCurrentVideoFrame), UIUpdateTimer); //initialize Remote emotiog Detection TimeSpan EmotionUpdateTimer = TimeSpan.FromMilliseconds(2000); // every 2 seconds this.EmotionProcessingTimer = Windows.System.Threading.ThreadPoolTimer.CreatePeriodicTimer(new Windows.System.Threading.TimerElapsedHandler(AnalyzeEmotion), EmotionUpdateTimer); }
private async Task InitFacialeRecon() { // Creates the Face tracker object this.faceTracker = await FaceTracker.CreateAsync(); // Set the frame rate TimeSpan timerInterval = TimeSpan.FromMilliseconds(66); // 15 fps Debug.WriteLine("Face tracker initializating"); this.frameProcessingTimer = Windows.System.Threading.ThreadPoolTimer.CreatePeriodicTimer(new Windows.System.Threading.TimerElapsedHandler(ProcessCurrentVideoFrame), timerInterval); Debug.WriteLine("Face tracker initializated !"); // Gets the video properties var deviceController = this._mediaCapture.VideoDeviceController; this.videoProperties = deviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; // Process frames with the tracker ProcessCurrentVideoFrame(frameProcessingTimer); }
private void Run() { //initialize FaceTracker faceTracker = new FaceTracker(tracker_model_json_filepath); //initialize FaceTrackerParams faceTrackerParams = new FaceTrackerParams(); cascade = new CascadeClassifier(); cascade.load(haarcascade_frontalface_alt_xml_filepath); // if (cascade.empty()) // { // Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); // } #if UNITY_ANDROID && !UNITY_EDITOR // Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2). webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true; #endif webCamTextureToMatHelper.Initialize(); }
public TrackerVisualizer(FaceTracker tracker) { this.tracker = tracker; Add(box = new Box { Colour = Colour4.Red, Alpha = 0.5f }); for (int i = 0; i < 73; i++) { var circle = new Circle { Size = new Vector2(2.5f), Colour = Colour4.Blue, Alpha = 0.0f }; circles.Add(circle); Add(circle); } }
public void StartKinectST() { foreach (double filterValue in gaussFilter) { gaussFactor += filterValue; } Kinect = KinectSensor.KinectSensors.FirstOrDefault(s => s.Status == KinectStatus.Connected); // Get first Kinect Sensor Kinect.SkeletonStream.Enable(); // Enable skeletal tracking Kinect.ColorStream.Enable(); Kinect.DepthStream.Enable(); SkeletonData = new Skeleton[Kinect.SkeletonStream.FrameSkeletonArrayLength]; // Allocate ST data Kinect.Start(); // Start Kinect sensor faceTracker = new FaceTracker(Kinect); Kinect.AllFramesReady += this.OnAllFramesReady; //Set Near and Seated Mode Kinect.SkeletonStream.EnableTrackingInNearRange = true; Kinect.DepthStream.Range = DepthRange.Near; Kinect.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated; }
public TestSceneFaceTracking() { Children = new Drawable[] { tracker = new FaceTracker(), camera = new CameraSprite { CameraID = 0, }, faceLocationsContainer = new Container <TrackingBox> { Name = @"face locations", Size = new Vector2(640, 480), }, new Container { AutoSizeAxes = Axes.Both, Children = new Drawable[] { new Box { Colour = Colour4.Black, RelativeSizeAxes = Axes.Both, }, status = new SpriteText { AlwaysPresent = true, Margin = new MarginPadding(5) }, } } }; tracker.StartTracking(camera); tracker.OnTrackerUpdate += _ => { trackerDeltaTime = Time.Current - lastTrackingTime; lastTrackingTime = Time.Current; }; }
/// <summary> /// Updates the face tracking information for this skeleton /// </summary> internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest) { this.skeletonTrackingState = skeletonOfInterest.TrackingState; if (this.skeletonTrackingState != SkeletonTrackingState.Tracked) { // nothing to do with an untracked skeleton. return; } if (this.faceTracker == null) { try { this.faceTracker = new FaceTracker(kinectSensor); } catch (InvalidOperationException) { this.faceTracker = null; } } if (this.faceTracker != null) { frame = this.faceTracker.Track( colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest).Clone() as FaceTrackFrame; this.lastFaceTrackSucceeded = frame.TrackSuccessful; if (this.lastFaceTrackSucceeded) { /*if (faceTriangles == null) * { * // only need to get this once. It doesn't change. * faceTriangles = frame.GetTriangles(); * } * * this.facePoints = frame.GetProjected3DShape();*/ } } }
private async void InitializeFacialRecognition() { if (m_faceTracker == null) { m_faceTracker = await FaceTracker.CreateAsync(); } m_mediaCapture = new MediaCapture(); MediaCaptureInitializationSettings settings = new MediaCaptureInitializationSettings(); settings.StreamingCaptureMode = StreamingCaptureMode.Video; await m_mediaCapture.InitializeAsync(settings); VideoDeviceController deviceController = m_mediaCapture.VideoDeviceController; m_videoProperties = deviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; await m_mediaCapture.StartPreviewAsync(); TimeSpan timerInterval = TimeSpan.FromMilliseconds(66); m_frameProcessingTimer = ThreadPoolTimer.CreatePeriodicTimer(new TimerElapsedHandler(ProcessCurrentVideoFrame), timerInterval); }
/// <summary> /// Face rotation display angle increment in degrees /// </summary> // const double FaceRotationIncrementInDegrees = 0.1; public MainWindow() { // 色々よしなにしてくれるおまじない. InitializeComponent(); try { // kinect is connected? if (KinectSensor.KinectSensors.Count == 0) { // 例外処理 MessageBox.Show("No KinectSensors ditected"); Close(); } // 1個目のKinectのKinectsensor instanceを取得, kinectに格納 // 複数のときはforeach文が便利. その場合はthisを使う(?) kinect = KinectSensor.KinectSensors[0]; // Color,Depth,Skeltonを有効化 kinect.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30); kinect.DepthStream.Enable(DepthImageFormat.Resolution640x480Fps30); kinect.SkeletonStream.Enable(); // kinect.ColorStream.Enable(ColorImageFormat.InfraredResolution640x480Fps30); // すべての情報の更新について実行されるイベント kinect.AllFramesReady += new EventHandler <AllFramesReadyEventArgs>(Kinect_AllFramesReady); // 動作開始 kinect.Start(); // 顔追跡用インスタンス生成 faceTracker = new FaceTracker(kinect); } catch (Exception ex) { MessageBox.Show(ex.Message); Close(); } }
private void Init() { try { string[] splitString = { ", " }; gaussFactor = 0; gaussFilter.Clear(); string[] filterValues = tbSmoothingFilter.Text.Split(splitString, StringSplitOptions.None); foreach (var filterValue in filterValues) { int value = Int32.Parse(filterValue); gaussFilter.Add(value); gaussFactor += value; } Kinect = KinectSensor.KinectSensors.FirstOrDefault(s => s.Status == KinectStatus.Connected); // Get first Kinect Sensor Kinect.SkeletonStream.Enable(); // Enable skeletal tracking Kinect.ColorStream.Enable(); Kinect.DepthStream.Enable(); SkeletonData = new Skeleton[Kinect.SkeletonStream.FrameSkeletonArrayLength]; // Allocate ST data Kinect.Start(); // Start Kinect sensor faceTracker = new FaceTracker(Kinect); Kinect.AllFramesReady += this.OnAllFramesReady; //Set Near and Seated Mode //Not available for Xbox Sensor! Kinect.SkeletonStream.EnableTrackingInNearRange = true; Kinect.DepthStream.Range = DepthRange.Near; Kinect.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated; File.AppendAllText("init.txt", DateTime.Now + " - Kinect sensor initialized successfully.\n"); useSpeach = InitSpeechRecognition(); } catch (Exception e) { System.Windows.MessageBox.Show("Error during Kinect Initialization. Ensure that Kinect sensor is connected correctly.\n\nError Message:\n" + e.ToString()); File.AppendAllText("init.txt", DateTime.Now + " - Error during Kinect initialization.\n"); } }
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest) { this.skeletonTrackingState = skeletonOfInterest.TrackingState; if (this.skeletonTrackingState != SkeletonTrackingState.Tracked) { return; } if (this.faceTracker == null) { try { this.faceTracker = new FaceTracker(kinectSensor); } catch (InvalidOperationException) { Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException"); this.faceTracker = null; } } if (this.faceTracker != null) { FaceTrackFrame frame = this.faceTracker.Track(colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest); this.lastFaceTrackSucceeded = frame.TrackSuccessful; if (this.lastFaceTrackSucceeded) { if (faceTriangles == null) { faceTriangles = frame.GetTriangles(); } this.facePoints = frame.GetProjected3DShape(); } } }
protected override async void OnNavigatedTo(NavigationEventArgs e) { _state = StreamingState.Idle; if (_faceTracker == null) { _faceTracker = await FaceTracker.CreateAsync(); } if (_faceApiHelper == null) { try { _faceApiHelper = new FaceApiHelper(); await _faceApiHelper.CheckGroupExistAsync(); } catch (Microsoft.ProjectOxford.Face.FaceAPIException faceEx) { ShowAlertHelper.ShowDialog(faceEx.ErrorMessage, faceEx.ErrorCode); } catch (Exception ex) { ShowAlertHelper.ShowDialog(ex.Message); } } if (_dataHelper == null) { try { _dataHelper = new DataHelper(); } catch (Exception ex) { ShowAlertHelper.ShowDialog(ex.Message); } } }
private async Task <bool> StartWebcamStreamingAsync() { _faceTracker = await FaceTracker.CreateAsync(); try { _mediaCapture = new MediaCapture(); await _mediaCapture.InitializeAsync(new MediaCaptureInitializationSettings { StreamingCaptureMode = StreamingCaptureMode.Video }); _mediaCapture.Failed += (s, a) => AbandonStreaming(); var deviceController = _mediaCapture.VideoDeviceController; _videoProperties = deviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; CameraPreview.Source = _mediaCapture; await _mediaCapture.StartPreviewAsync(); var timerInterval = TimeSpan.FromMilliseconds(66); // 66ms, aprox 15 fps _frameProcessingTimer = ThreadPoolTimer.CreatePeriodicTimer(ProcessCurrentVideoFrame, timerInterval); return(true); } catch (UnauthorizedAccessException) { NavigateToPermissionsPage(); return(false); } catch (Exception exception) { await DisplayMessage($"Error al iniciar el stream de la cámara: {exception.Message}"); return(false); } }
public void init(Context context, FaceEnvironment environment) { this.context = context; this.faceEnvironment = environment; if (mFaceTracker == null) { mFaceTracker = new FaceTracker(context); mFaceTracker.Set_isFineAlign(false); mFaceTracker.Set_isVerifyLive(false); mFaceTracker.Set_isCheckQuality(environment.isCheckQuality()); mFaceTracker.Set_DetectMethodType(1); mFaceTracker.Set_isCheckQuality(environment.isCheckQuality()); mFaceTracker.Set_notFace_thr(environment.getNotFaceThreshold()); mFaceTracker.Set_min_face_size(environment.getMiniFaceSize()); mFaceTracker.Set_cropFaceSize(FaceEnvironment.VALUE_CROP_FACE_SIZE); mFaceTracker.Set_illum_thr(environment.getIlluminationThreshold()); mFaceTracker.Set_blur_thr(environment.getBlurrinessThreshold()); mFaceTracker.Set_occlu_thr(environment.getOcclulationThreshold()); mFaceTracker.Set_max_reg_img_num(FaceEnvironment.VALUE_MAX_CROP_IMAGE_NUM); mFaceTracker.Set_eulur_angle_thr(environment.getPitch(), environment.getYaw(), environment.getRoll()); // mFaceTracker.set_track_by_detection_interval(50); } }
private void Run() { //set 3d face object points. objectPoints = new MatOfPoint3f(new Point3(-31, 72, 86), //l eye new Point3(31, 72, 86), //r eye new Point3(0, 40, 114), //nose new Point3(-20, 15, 90), //l mouse new Point3(20, 15, 90) //r mouse // , // new Point3 (-70, 60, -9),//l ear // new Point3 (70, 60, -9)//r ear ); imagePoints = new MatOfPoint2f(); rvec = new Mat(); tvec = new Mat(); rotM = new Mat(3, 3, CvType.CV_64FC1); //initialize FaceTracker faceTracker = new FaceTracker(tracker_model_json_filepath); //initialize FaceTrackerParams faceTrackerParams = new FaceTrackerParams(); cascade = new CascadeClassifier(); cascade.load(haarcascade_frontalface_alt_xml_filepath); // if (cascade.empty()) // { // Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); // } #if UNITY_ANDROID && !UNITY_EDITOR // Avoids the front camera low light issue that occurs in only some Android devices (e.g. Google Pixel, Pixel2). webCamTextureToMatHelper.avoidAndroidFrontCameraLowLightIssue = true; #endif webCamTextureToMatHelper.Initialize(); }
/// <summary> /// Responds when we navigate to this page. /// </summary> /// <param name="e">Event data</param> protected override async void OnNavigatedTo(NavigationEventArgs e) { if (ApplicationData.Current.LocalSettings.Values.ContainsKey("ApiKey")) { ApiKey.Text = ApplicationData.Current.LocalSettings.Values["ApiKey"].ToString(); } if (ApplicationData.Current.LocalSettings.Values.ContainsKey("ApiEndPoint")) { ApiEndPoint.Text = ApplicationData.Current.LocalSettings.Values["ApiEndPoint"].ToString(); } if (this.faceTracker == null) { this.faceTracker = await FaceTracker.CreateAsync(); } if (this.faceClient == null) { this.faceClient = new FaceClient( new ApiKeyServiceClientCredentials(ApiKey.Text), new System.Net.Http.DelegatingHandler[] { }) { Endpoint = ApiEndPoint.Text }; } if (captureFolder == null) { var picturesLibrary = await StorageLibrary.GetLibraryAsync(KnownLibraryId.Pictures); // Fall back to the local app storage if the Pictures Library is not available captureFolder = picturesLibrary.SaveFolder ?? ApplicationData.Current.LocalFolder; } await this.StartWebcamStreaming(); }
/// <summary> /// Selects the FeaturePoints of all tracked skeletons from the source observable. /// </summary> /// <param name="observable">The source observable.</param> /// <param name="faceTracker">The FaceTracker that is used to track the faces.</param> /// <returns>A sequence of a collection of FeaturePoints and their identifiers in a tuple.</returns> public static IObservable <IEnumerable <Tuple <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> > > > SelectPersonPoints(this IObservable <Tuple <ColorImageFormat, byte[], DepthImageFormat, short[], Skeleton[]> > observable, FaceTracker faceTracker) { if (observable == null) { throw new ArgumentNullException("observable"); } if (faceTracker == null) { throw new ArgumentNullException("faceTracker"); } return(observable.Select(_ => _.Item5.ForEach <Skeleton, Tuple <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> > >(__ => { if (__.TrackingState == SkeletonTrackingState.PositionOnly) { return Tuple.Create <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> >(__.TrackingId, __.TrackingState, __.Joints, null); } var faceTrackFrame = faceTracker.Track(_.Item1, _.Item2, _.Item3, _.Item4, __); if (!faceTrackFrame.TrackSuccessful) { return Tuple.Create <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> >(__.TrackingId, __.TrackingState, __.Joints, null); } return Tuple.Create(__.TrackingId, __.TrackingState, __.Joints, faceTrackFrame.GetProjected3DShape()); }))); }
public CalibrationCompletedDataSender(IMessageSender sender, FaceTracker faceTracker) { faceTracker.CalibrationCompleted += data => sender.SendCommand(MessageFactory.Instance.SetCalibrationFaceData(data)); }
public FaceTrackerProxy (Canvas canvas, MainPage page, CaptureElement capture, MediaCapture mediacapture ) { if (this.faceTracker == null) { this.faceTracker = FaceTracker.CreateAsync().AsTask().Result; } rootPage = page; VisualizationCanvas = canvas; this.VisualizationCanvas.Children.Clear(); mediaCapture = mediacapture; var deviceController = mediaCapture.VideoDeviceController; this.videoProperties = deviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; currentState = ScenarioState.Streaming; // Ensure the Semaphore is in the signalled state. this.frameProcessingSemaphore.Release(); // Use a 66 milisecond interval for our timer, i.e. 15 frames per second TimeSpan timerInterval = TimeSpan.FromMilliseconds(200); this.frameProcessingTimer = Windows.System.Threading.ThreadPoolTimer.CreatePeriodicTimer(new Windows.System.Threading.TimerElapsedHandler(ProcessCurrentVideoFrame), timerInterval); }
// Use this for initialization void Start() { //initialize FaceTracker faceTracker = new FaceTracker (Utils.getFilePath ("tracker_model.json")); //initialize FaceTrackerParams faceTrackerParams = new FaceTrackerParams (); StartCoroutine (init ()); }
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest) { if (!timerStarted) { timer.Start(); timerStarted = true; } //increment our frames numberOfFrames++; this.skeletonTrackingState = skeletonOfInterest.TrackingState; if (this.skeletonTrackingState != SkeletonTrackingState.Tracked) { // nothing to do with an untracked skeleton. return; } if (this.faceTracker == null) { try { this.faceTracker = new FaceTracker(kinectSensor); } catch (InvalidOperationException) { // During some shutdown scenarios the FaceTracker // is unable to be instantiated. Catch that exception // and don't track a face. Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException"); this.faceTracker = null; } } if (this.faceTracker != null) { FaceTrackFrame frame = this.faceTracker.Track( colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest); this.lastFaceTrackSucceeded = frame.TrackSuccessful; if (this.lastFaceTrackSucceeded) { if (faceTriangles == null) { // only need to get this once. It doesn't change. faceTriangles = frame.GetTriangles(); } this.facePoints = frame.GetProjected3DShape(); /*if () * { * Debug.WriteLine("hit " + (frameIter * sampleRate) + " frames in " + (timer.Elapsed) + " seconds"); * frameIter++; * }*/ //Also grab our points EnumIndexableCollection <FeaturePoint, Vector3DF> facePoints3D = frame.Get3DShape(); int index = 0; if (numberOfFrames > frameIter * sampleRate && frameIter < 5) //only grab 4 samples over our given sample rate { //Create a new thread so we don't make the visual thread throw up all over the place new Thread(() => { Thread.CurrentThread.IsBackground = true; List <Tuple <float, float, float> > myPoints = new List <Tuple <float, float, float> >(); foreach (Vector3DF vector in facePoints3D) { //csv.Append(string.Format("( ({1}, {2}, {3}){4}",vector.X, vector.Y, vector.Z, Environment.NewLine)); myPoints.Add(new Tuple <float, float, float>(vector.X, vector.Y, vector.Z)); index++; } calculateDistances(myPoints); frameIter++; }).Start(); //once = true; } if (frameIter == 5) { SetStatusText("Generating histograms..."); Console.WriteLine("We are ready to sample"); foreach (float distance in sampleOneDistances) { int sampleOneIndex = (int)Math.Floor(64 * distance / sampleOneMaxDistance); sampleOneHistogram[sampleOneIndex]++; } foreach (float distance in sampleTwoDistances) { sampleTwoHistogram[(int)Math.Floor(64 * distance / sampleTwoMaxDistance)]++; } foreach (float distance in sampleThreeDistances) { sampleThreeHistogram[(int)Math.Floor(64 * distance / sampleThreeMaxDistance)]++; } foreach (float distance in sampleFourDistances) { sampleFourHistogram[(int)Math.Floor(64 * distance / sampleFourMaxDistance)]++; } //Go through histogram and divide by distances //Get for (int i = 0; i < sampleOneHistogram.Length; i++) { sampleOneHistogram[i] = sampleOneHistogram[i] / sampleOneDistances.Count; } for (int i = 0; i < sampleTwoHistogram.Length; i++) { sampleTwoHistogram[i] = sampleTwoHistogram[i] / sampleTwoDistances.Count; } for (int i = 0; i < sampleThreeHistogram.Length; i++) { sampleThreeHistogram[i] = sampleThreeHistogram[i] / sampleThreeDistances.Count; } for (int i = 0; i < sampleFourHistogram.Length; i++) { sampleFourHistogram[i] = sampleFourHistogram[i] / sampleFourDistances.Count; } int iter = 0; foreach (int count in sampleTwoHistogram)//can iterate through any histogram, they're all of size 65 { Console.WriteLine("Count for hist1/2/3/4[" + iter + "] is " + count + "/" + sampleOneHistogram[iter] + "/" + sampleThreeHistogram[iter] + "/" + sampleFourHistogram[iter]); iter++; } //Write our histograms to a csv file String[] sampleOneHistString = Array.ConvertAll(sampleOneHistogram, x => x.ToString()); using (System.IO.StreamWriter file = new System.IO.StreamWriter(testFilePath)) { file.Write(string.Join(",", Enumerable.Range(1, 65).ToArray()) + Environment.NewLine); file.Write(string.Join(",", sampleOneHistString)); file.Write(Environment.NewLine); file.Write(string.Join(",", Array.ConvertAll(sampleTwoHistogram, x => x.ToString()))); file.Write(Environment.NewLine); file.Write(string.Join(",", Array.ConvertAll(sampleThreeHistogram, x => x.ToString()))); file.Write(Environment.NewLine); file.Write(string.Join(",", Array.ConvertAll(sampleFourHistogram, x => x.ToString()))); } //pass that data file to jar String jarPath = "C:\\Users\\Datalab\\Documents\\GitHub\\WekaClassifier\\jar\\wekaClassifier.jar"; System.Diagnostics.Process clientProcess = new Process(); String jarargs = "C:\\Users\\Datalab\\Documents\\GitHub\\WekaClassifier\\data\\training_data.arff C:\\Users\\Datalab\\Documents\\GitHub\\WekaClassifier\\data\\testFormat.dat"; clientProcess.StartInfo.FileName = "java"; clientProcess.StartInfo.Arguments = "-jar " + jarPath + " " + jarargs; clientProcess.StartInfo.RedirectStandardOutput = true; clientProcess.StartInfo.UseShellExecute = false; clientProcess.Start(); String output = clientProcess.StandardOutput.ReadToEnd(); Console.WriteLine(output); clientProcess.WaitForExit(); int code = clientProcess.ExitCode; //write to dat file with 4 histograms averaged frameIter++; //only do this once (will make conditional evaluate to false. Is this clean and clear? Not really? Do I care? Not particularly. At least it's documented. ftNumPeople++; SetPeopleText("People tracked : " + ftNumPeople); SetStatusText("Status: waiting...."); SetPredictionText("Guess: " + output); } } } }
public async Task StartStreamAsync(bool restarted = false, bool isForRealTimeProcessing = false) { try { if (captureManager == null || captureManager.CameraStreamState == CameraStreamState.Shutdown) { captureManager = new MediaCapture(); MediaCaptureInitializationSettings settings = new MediaCaptureInitializationSettings(); var allCameras = await DeviceInformation.FindAllAsync(DeviceClass.VideoCapture); var selectedCamera = allCameras.FirstOrDefault(c => c.Name == SettingsHelper.Instance.CameraName); if (selectedCamera != null) { settings.VideoDeviceId = selectedCamera.Id; } await captureManager.InitializeAsync(settings); await SetVideoEncodingToHighestResolution(isForRealTimeProcessing); this.webCamCaptureElement.Source = captureManager; } if (captureManager.CameraStreamState == CameraStreamState.NotStreaming) { if (this.faceTracker == null) { this.faceTracker = await FaceTracker.CreateAsync(); } this.videoProperties = this.captureManager.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; await captureManager.StartPreviewAsync(); if (this.frameProcessingTimer != null) { this.frameProcessingTimer.Stop(); frameProcessingSemaphore.Release(); } TimeSpan timerInterval = TimeSpan.FromMilliseconds(66); //15fps // this.frameProcessingTimer = ThreadPoolTimer.CreatePeriodicTimer(new TimerElapsedHandler(ProcessCurrentVideoFrame), timerInterval); DispatcherTimer tim = new DispatcherTimer { Interval = timerInterval }; tim.Tick += (a, b) => { ProcessCurrentVideoFrame(tim); }; this.frameProcessingTimer = tim; // new DispatcherTimer { Interval = timerInterval }; // ThreadPoolTimer.CreatePeriodicTimer(new TimerElapsedHandler(ProcessCurrentVideoFrame), timerInterval); // this.cameraControlSymbol.Symbol = Symbol.Camera; tim.Start(); this.webCamCaptureElement.Visibility = Visibility.Visible; } this.captureManager.Failed += CaptureManager_Failed; if (restarted) { this.CameraRestarted.Invoke(this, EventArgs.Empty); } } catch (Exception ex) { await Util.GenericApiCallExceptionHandler(ex, "Error starting the camera."); } }
/// <summary> /// Track a face and update the states. /// </summary> /// <param name="sensor">Instance of KinectSensor</param> /// <param name="colorImageFormat">Format of the colorImage array</param> /// <param name="colorImage">Input color image frame retrieved from Kinect sensor</param> /// <param name="depthImageFormat">Format of the depthImage array</param> /// <param name="depthImage">Input depth image frame retrieved from Kinect sensor</param> /// <param name="skeletonOfInterest">Input skeleton to track. Head and shoulder joints in the skeleton are used to calculate the head vector</param> /// <param name="computedBoundingBox">Whether compute the bounding box of the face mask</param> public void TrackFace( KinectSensor sensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest, bool computedBoundingBox) { // Reset the valid flag this.faceInfo.TrackValid = false; if (null == this.faceTracker) { try { this.faceTracker = new FaceTracker(sensor); } catch (InvalidOperationException) { // Fail silently this.faceTracker = null; return; } } // Set the color image width Size colorImageSize = Helper.GetImageSize(colorImageFormat); this.colorWidth = colorImageSize.Width; // Track the face and update the states if (this.faceTracker != null && skeletonOfInterest != null && skeletonOfInterest.TrackingState == SkeletonTrackingState.Tracked) { FaceTrackFrame faceTrackFrame = this.faceTracker.Track( colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest); this.faceInfo.TrackValid = faceTrackFrame.TrackSuccessful; if (this.faceInfo.TrackValid) { this.faceInfo.FaceRect = faceTrackFrame.FaceRect; this.faceInfo.Rotation = faceTrackFrame.Rotation; this.faceInfo.Translation = faceTrackFrame.Translation; // Get the bounding box of face mask if (computedBoundingBox) { var shapePoints = faceTrackFrame.Get3DShape(); this.ResetBoundingBox(); // Reset the minimum and maximum points of bounding box foreach (var point in shapePoints) { if (point.X < this.minimumPoint.X) { this.minimumPoint.X = point.X; } if (point.X > this.maximumPoint.X) { this.maximumPoint.X = point.X; } if (point.Y < this.minimumPoint.Y) { this.minimumPoint.Y = point.Y; } if (point.Y > this.maximumPoint.Y) { this.maximumPoint.Y = point.Y; } if (point.Z < this.minimumPoint.Z) { this.minimumPoint.Z = point.Z; } if (point.Z > this.maximumPoint.Z) { this.maximumPoint.Z = point.Z; } } } } } // To render the face rectangle Dispatcher.BeginInvoke((Action)(() => this.InvalidateVisual())); }
void KinectFaceNode_AllFrameReady(object sender, AllFramesReadyEventArgs e) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; if (face == null) { face = new FaceTracker(this.runtime.Runtime); } colorImageFrame = e.OpenColorImageFrame(); depthImageFrame = e.OpenDepthImageFrame(); skeletonFrame = e.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { return; } if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } colorImageFrame.CopyPixelDataTo(this.colorImage); depthImageFrame.CopyPixelDataTo(this.depthImage); skeletonFrame.CopySkeletonDataTo(this.skeletonData); foreach (Skeleton skeleton in this.skeletonData) { if (skeleton.TrackingState == SkeletonTrackingState.Tracked || skeleton.TrackingState == SkeletonTrackingState.PositionOnly) { // We want keep a record of any skeleton, tracked or untracked. if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId)) { this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker()); } // Give each tracker the upated frame. SkeletonFaceTracker skeletonFaceTracker; if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker)) { skeletonFaceTracker.OnFrameReady(this.runtime.Runtime, ColorImageFormat.RgbResolution640x480Fps30, colorImage, DepthImageFormat.Resolution320x240Fps30, depthImage, skeleton); skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber; } } } this.RemoveOldTrackers(skeletonFrame.FrameNumber); colorImageFrame.Dispose(); depthImageFrame.Dispose(); skeletonFrame.Dispose(); this.FInvalidate = true; }
/// <summary> /// Creates new face tracker /// Recycles inactive trackers /// </summary> /// <param name="trackingRegion">Region of image to track</param> /// <param name="xScale">X scale of downsampled image to original image</param> /// <param name="yScale">Y scale if downsampled image to original image</param> /// <param name="image">Downsampled image from which we trace given rectangle</param> private void CreateFaceTracker(Rectangle trackingRegion, double xScale, double yScale, UnmanagedImage image) { // Find first not active tracker // object or create new one if none found int faceTrackerIndex = -1; for (int index = 0; index < this.trackers.Count; index++) { if (!this.trackers.ElementAt(index).IsActive) { faceTrackerIndex = index; break; } } FaceTracker faceTracker; if (faceTrackerIndex < 0) { faceTracker = new FaceTracker(); this.trackers.Add(faceTracker); } else faceTracker = this.trackers.ElementAt(faceTrackerIndex); // Reduce the face size to avoid tracking background var trackingArea = new Rectangle( (int)((trackingRegion.X + trackingRegion.Width / 2f) * xScale), (int)((trackingRegion.Y + trackingRegion.Height / 2f) * yScale), 1, 1); trackingArea.Inflate( (int)(0.25f * trackingRegion.Width * xScale), (int)(0.40f * trackingRegion.Height * yScale)); // Apply tracking area to tracker object faceTracker.IsActive = true; faceTracker.Tracker.Reset(); faceTracker.Tracker.SearchWindow = trackingArea; faceTracker.Tracker.ProcessFrame(image); }
public async Task StartStreamAsync(bool isForRealTimeProcessing = false, DeviceInformation desiredCamera = null) { try { if (captureManager == null || captureManager.CameraStreamState == CameraStreamState.Shutdown || captureManager.CameraStreamState == CameraStreamState.NotStreaming) { loadingOverlay.Visibility = Visibility.Visible; if (captureManager != null) { captureManager.Dispose(); } captureManager = new MediaCapture(); MediaCaptureInitializationSettings settings = new MediaCaptureInitializationSettings(); var allCameras = await DeviceInformation.FindAllAsync(DeviceClass.VideoCapture); var selectedCamera = allCameras.FirstOrDefault(c => c.Name == SettingsHelper.Instance.CameraName); if (desiredCamera != null) { selectedCamera = desiredCamera; } else if (lastUsedCamera != null) { selectedCamera = lastUsedCamera; } if (selectedCamera != null) { settings.VideoDeviceId = selectedCamera.Id; lastUsedCamera = selectedCamera; } cameraSwitchButton.Visibility = allCameras.Count > 1 ? Visibility.Visible : Visibility.Collapsed; await captureManager.InitializeAsync(settings); await SetVideoEncodingToHighestResolution(isForRealTimeProcessing); isStreamingOnRealtimeResolution = isForRealTimeProcessing; this.webCamCaptureElement.Source = captureManager; } if (captureManager.CameraStreamState == CameraStreamState.NotStreaming) { if (PerformFaceTracking || CameraFrameProcessor != null) { if (this.faceTracker == null) { this.faceTracker = await FaceTracker.CreateAsync(); } if (this.frameProcessingTimer != null) { this.frameProcessingTimer.Cancel(); frameProcessingSemaphore.Release(); } TimeSpan timerInterval = TimeSpan.FromMilliseconds(66); //15fps this.frameProcessingTimer = ThreadPoolTimer.CreatePeriodicTimer(new TimerElapsedHandler(ProcessCurrentVideoFrame), timerInterval); } this.videoProperties = this.captureManager.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties; await captureManager.StartPreviewAsync(); this.webCamCaptureElement.Visibility = Visibility.Visible; loadingOverlay.Visibility = Visibility.Collapsed; } } catch (Exception ex) { await Util.GenericApiCallExceptionHandler(ex, "Error starting the camera."); } }
/// <summary> /// Processes given frame for given face tracker /// </summary> /// <param name="faceTracker">Face tracker to process</param> /// <param name="image">Frame to process for tracker</param> /// <returns>Returns asynchronous void Task</returns> private async Task TrackFaceAsync(FaceTracker faceTracker, UnmanagedImage image) { // Check if face tracker is active if (!faceTracker.IsActive) return; // Process given frame await Task.Run(() => faceTracker.Tracker.ProcessFrame(image)); // Get object properties var trackingObject = faceTracker.Tracker.TrackingObject; // If tracking object is empty, start new detection if (trackingObject.IsEmpty || trackingObject.Area < 5) { faceTracker.IsActive = false; this.log.Info("Face not in view, tracker set to inactive state"); return; } }
//private async void ProcessCurrentVideoFrame(ThreadPoolTimer timer) private async Task <int> ProcessCurrentVideoFrame() { // If a lock is being held it means we're still waiting for processing work on the previous frame to complete. // In this situation, don't wait on the semaphore but exit immediately. /*if (!frameProcessingSemaphore.Wait(0)) * { * Debug.WriteLine("\t --> ProcessCurrentFrame call blocked !"); * return 0; * }*/ Debug.WriteLine("\t --> ProcessCurrentFrame called properly!"); // List of detected faces /* if (videoProcessor.GetLatestFrame() != null) * {*/ Debug.WriteLine("\t --> calling videoProcessor.GetLatestFrame() !"); frame = videoProcessor.GetLatestFrame(); // frame = videoProcessor.latestFrame; /* } * else * { * //frameProcessingSemaphore.Release(); * return 0; * }*/ Debug.WriteLine("\t --> videoProcessor.GetLatestFrame() called !"); if (frame != null) { if (FaceTracker.IsBitmapPixelFormatSupported(frame.VideoMediaFrame.SoftwareBitmap.BitmapPixelFormat)) { Debug.WriteLine("\t --> Format: OK!"); // this.faces = await this.faceTracker.ProcessNextFrameAsync(frame.VideoMediaFrame.GetVideoFrame()); var faceTask = this.faceTracker.ProcessNextFrameAsync(frame.VideoMediaFrame.GetVideoFrame()); this.faces = faceTask.GetResults(); Debug.WriteLine("\t --> Frame processed!"); } else { Debug.WriteLine("\t--> Format : NOT OK!"); } } else { Debug.WriteLine("\t --> last frame was null !"); return(0); } if (this.faces.Count == 0) { this.faces.Clear(); Debug.WriteLine("No Face detected"); //FaceCoord.x = "0"; // FaceCoord.y = "0"; } else { Debug.WriteLine("Face detected"); latestfaces.Clear(); foreach (DetectedFace face in faces) { Debug.WriteLine("faces size: " + faces.Count.ToString()); latestfaces.Add(face.FaceBox); } foreach (BitmapBounds latestface in latestfaces) { Debug.WriteLine("faces size: " + latestfaces.Count.ToString()); Debug.WriteLine("faceX" + latestface.X.ToString()); Debug.WriteLine("faceY" + latestface.Y.ToString()); // FaceCoord.x = latestface.X.ToString(); // FaceCoord.y = latestface.Y.ToString(); } } // frameProcessingSemaphore.Release(); return(0); }
private void AllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; try { colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { return; } // Check for changes in any of the data this function is receiving // and reset things appropriately. if (this.depthImageFormat != depthImageFrame.Format) { this.DestroyFaceTracker(); this.depthImage = null; this.depthImageFormat = depthImageFrame.Format; } if (this.colorImageFormat != colorImageFrame.Format) { this.DestroyFaceTracker(); this.colorImage = null; this.colorImageFormat = colorImageFrame.Format; this.colorImageWritableBitmap = null; this.ColorImage.Source = null; this.theMaterial.Brush = null; } if (this.skeletonData != null && this.skeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.skeletonData = null; } // Create any buffers to store copies of the data we work with if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } if (this.colorImageWritableBitmap == null) { this.colorImageWritableBitmap = new WriteableBitmap( colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null); this.ColorImage.Source = this.colorImageWritableBitmap; this.theMaterial.Brush = new ImageBrush(this.colorImageWritableBitmap) { ViewportUnits = BrushMappingMode.Absolute }; } if (this.skeletonData == null) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } // Copy data received in this event to our buffers. colorImageFrame.CopyPixelDataTo(this.colorImage); depthImageFrame.CopyPixelDataTo(this.depthImage); skeletonFrame.CopySkeletonDataTo(this.skeletonData); this.colorImageWritableBitmap.WritePixels( new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height), this.colorImage, colorImageFrame.Width * Bgr32BytesPerPixel, 0); // Find a skeleton to track. // First see if our old one is good. // When a skeleton is in PositionOnly tracking state, don't pick a new one // as it may become fully tracked again. Skeleton skeletonOfInterest = this.skeletonData.FirstOrDefault( skeleton => skeleton.TrackingId == this.trackingId && skeleton.TrackingState != SkeletonTrackingState.NotTracked); if (skeletonOfInterest == null) { // Old one wasn't around. Find any skeleton that is being tracked and use it. skeletonOfInterest = this.skeletonData.FirstOrDefault( skeleton => skeleton.TrackingState == SkeletonTrackingState.Tracked); if (skeletonOfInterest != null) { // This may be a different person so reset the tracker which // could have tuned itself to the previous person. if (this.faceTracker != null) { this.faceTracker.ResetTracking(); } this.trackingId = skeletonOfInterest.TrackingId; } } bool displayFaceMesh = false; if (skeletonOfInterest != null && skeletonOfInterest.TrackingState == SkeletonTrackingState.Tracked) { if (this.faceTracker == null) { try { this.faceTracker = new FaceTracker(this.Kinect); } catch (InvalidOperationException) { // During some shutdown scenarios the FaceTracker // is unable to be instantiated. Catch that exception // and don't track a face. Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException"); this.faceTracker = null; } } if (this.faceTracker != null) { FaceTrackFrame faceTrackFrame = this.faceTracker.Track( this.colorImageFormat, this.colorImage, this.depthImageFormat, this.depthImage, skeletonOfInterest); if (faceTrackFrame.TrackSuccessful) { this.UpdateMesh(faceTrackFrame); // Only display the face mesh if there was a successful track. displayFaceMesh = true; } } } else { this.trackingId = -1; } this.viewport3d.Visibility = displayFaceMesh ? Visibility.Visible : Visibility.Hidden; } finally { if (colorImageFrame != null) { colorImageFrame.Dispose(); } if (depthImageFrame != null) { depthImageFrame.Dispose(); } if (skeletonFrame != null) { skeletonFrame.Dispose(); } } }
private bool CheckFace(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest) { this.skeletonTrackingState = skeletonOfInterest.TrackingState; if (this.skeletonTrackingState != SkeletonTrackingState.Tracked) { // nothing to do with an untracked skeleton. return(false); } if (this.faceTracker == null) { try { this.faceTracker = new FaceTracker(kinectSensor); } catch (InvalidOperationException) { // During some shutdown scenarios the FaceTracker // is unable to be instantiated. Catch that exception // and don't track a face. Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException"); this.faceTracker = null; } } if (this.faceTracker != null) { FaceTrackFrame frame = this.faceTracker.Track( colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest); this.lastFaceTrackSucceeded = frame.TrackSuccessful; if (this.lastFaceTrackSucceeded) { if (faceTriangles == null) { // only need to get this once. It doesn't change. faceTriangles = frame.GetTriangles(); } //getting the Animation Unit Coefficients this.AUs = frame.GetAnimationUnitCoefficients(); var jawLowerer = AUs[AnimationUnit.JawLower]; var browLower = AUs[AnimationUnit.BrowLower]; var browRaiser = AUs[AnimationUnit.BrowRaiser]; var lipDepressor = AUs[AnimationUnit.LipCornerDepressor]; var lipRaiser = AUs[AnimationUnit.LipRaiser]; var lipStretcher = AUs[AnimationUnit.LipStretcher]; //set up file for output using (System.IO.StreamWriter file = new System.IO.StreamWriter (@"C:\Users\Public\data.txt")) { file.WriteLine("FaceTrack Data, started recording at " + DateTime.Now.ToString("HH:mm:ss tt")); } //here is the algorithm to test different facial features //BrowLower is messed up if you wear glasses, works if you don't wear 'em string state = ""; //surprised if ((jawLowerer < 0.25 || jawLowerer > 0.25) && browLower < 0) { state = "surprised"; } //smiling if (lipStretcher > 0.4 || lipDepressor < 0) { state = "smiling"; } //sad if (browRaiser < 0 && lipDepressor > 0) { state = "sad"; } //angry if ((browLower > 0 && (jawLowerer > 0.25 || jawLowerer < -0.25)) || (browLower > 0 && lipDepressor > 0)) { state = "angry"; } //System.Diagnostics.Debug.WriteLine(browLower); this.facePoints = frame.GetProjected3DShape(); if (states[currentState] == state) { Trace.WriteLine("Yo!"); return(true); } } } return(false); }
private void Run() { gameObject.transform.localScale = new Vector3(imgTexture.width, imgTexture.height, 1); Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); float width = 0; float height = 0; width = gameObject.transform.localScale.x; height = gameObject.transform.localScale.y; float widthScale = (float)Screen.width / width; float heightScale = (float)Screen.height / height; if (widthScale < heightScale) { Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = height / 2; } //initialize FaceTracker FaceTracker faceTracker = new FaceTracker(tracker_model_json_filepath); //initialize FaceTrackerParams FaceTrackerParams faceTrackerParams = new FaceTrackerParams(); Mat imgMat = new Mat(imgTexture.height, imgTexture.width, CvType.CV_8UC4); Utils.texture2DToMat(imgTexture, imgMat); Debug.Log("imgMat dst ToString " + imgMat.ToString()); CascadeClassifier cascade = new CascadeClassifier(); cascade.load(haarcascade_frontalface_alt_xml_filepath); //if (cascade.empty()) //{ // Debug.LogError("cascade file is not loaded.Please copy from “FaceTrackerExample/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); //} //convert image to greyscale Mat gray = new Mat(); Imgproc.cvtColor(imgMat, gray, Imgproc.COLOR_RGBA2GRAY); MatOfRect faces = new MatOfRect(); Imgproc.equalizeHist(gray, gray); cascade.detectMultiScale(gray, faces, 1.1f, 2, 0 // | Objdetect.CASCADE_FIND_BIGGEST_OBJECT | Objdetect.CASCADE_SCALE_IMAGE, new Size(gray.cols() * 0.05, gray.cols() * 0.05), new Size()); Debug.Log("faces " + faces.dump()); if (faces.rows() > 0) { //add initial face points from MatOfRect faceTracker.addPoints(faces); } //track face points.if face points <= 0, always return false. if (faceTracker.track(imgMat, faceTrackerParams)) { faceTracker.draw(imgMat, new Scalar(255, 0, 0, 255), new Scalar(0, 255, 0, 255)); } Texture2D texture = new Texture2D(imgMat.cols(), imgMat.rows(), TextureFormat.RGBA32, false); Utils.matToTexture2D(imgMat, texture); gameObject.GetComponent <Renderer>().material.mainTexture = texture; cascade.Dispose(); }
/// <summary> /// Responds when we navigate to this page. /// </summary> /// <param name="e">Event data</param> protected override async void OnNavigatedTo(NavigationEventArgs e) { this.rootPage = MainPage.Current; // The 'await' operation can only be used from within an async method but class constructors // cannot be labeled as async, and so we'll initialize FaceTracker here. if (this.faceTracker == null) { this.faceTracker = await FaceTracker.CreateAsync(); } }