private async Task InitializeFaceDetection() { if (FaceDetector.IsSupported) { if (_faceDetector == null) { _faceDetector = await FaceDetector.CreateAsync(); _faceDectorSupportedPixelFormat = FaceDetector.GetSupportedBitmapPixelFormats().FirstOrDefault(); } } else { Debug.WriteLine("Face detection is not supported"); } if (FaceTracker.IsSupported) { if (_faceTracker == null) { _faceTracker = await FaceTracker.CreateAsync(); _faceTrackerSupportedPixelFormat = FaceTracker.GetSupportedBitmapPixelFormats().FirstOrDefault(); } } else { Debug.WriteLine("Face tracking is not suppoted"); } }
private async Task InitializeFaceDetection() { if (FaceDetector.IsSupported) { if (faceDetector == null) { faceDetector = await FaceDetector.CreateAsync(); faceDetectorSupportedPixelFormat = FaceDetector.GetSupportedBitmapPixelFormats().FirstOrDefault(); } } else { Debug.WriteLine("Warning. FaceDetector is not supported on this device"); } if (FaceTracker.IsSupported) { if (faceTracker == null) { faceTracker = await FaceTracker.CreateAsync(); faceTrackerSupportedPixelFormat = FaceTracker.GetSupportedBitmapPixelFormats().FirstOrDefault(); } } else { Debug.WriteLine("Warning. FaceTracking is not supported on this device"); } }
/// <summary> /// This is just one big lump of code right now which should be factored out into some kind of /// 'frame reader' class which can then be subclassed for depth frame and video frame but /// it was handy to have it like this while I experimented with it - the intention was /// to tidy it up if I could get it doing more or less what I wanted :-) /// </summary> async Task ProcessingLoopAsync() { var depthMediaCapture = await this.GetMediaCaptureForDescriptionAsync( MediaFrameSourceKind.Depth, 448, 450, 15); var depthFrameReader = await depthMediaCapture.Item1.CreateFrameReaderAsync(depthMediaCapture.Item2); depthFrameReader.AcquisitionMode = MediaFrameReaderAcquisitionMode.Realtime; MediaFrameReference lastDepthFrame = null; long depthFrameCount = 0; float centrePointDepthInMetres = 0.0f; // Expecting this to run at 1fps although the API (seems to) reports that it runs at 15fps TypedEventHandler <MediaFrameReader, MediaFrameArrivedEventArgs> depthFrameHandler = (sender, args) => { using (var depthFrame = sender.TryAcquireLatestFrame()) { if ((depthFrame != null) && (depthFrame != lastDepthFrame)) { lastDepthFrame = depthFrame; Interlocked.Increment(ref depthFrameCount); // Always try to grab the depth value although, clearly, this is subject // to a bunch of race conditions etc. as other thread access it. centrePointDepthInMetres = GetDepthValueAtCoordinate(depthFrame, (int)(depthFrame.Format.VideoFormat.Width * MAGIC_DEPTH_FRAME_WIDTH_RATIO_CENTRE), (int)(depthFrame.Format.VideoFormat.Height * MAGIC_DEPTH_FRAME_HEIGHT_RATIO_CENTRE)) ?? 0.0f; } } }; long rgbProcessedCount = 0; long facesPresentCount = 0; long rgbDroppedCount = 0; MediaFrameReference lastRgbFrame = null; var faceBitmapFormats = FaceTracker.GetSupportedBitmapPixelFormats().Select( format => format.ToString().ToLower()).ToArray(); var faceTracker = await FaceTracker.CreateAsync(); var rgbMediaCapture = await this.GetMediaCaptureForDescriptionAsync( MediaFrameSourceKind.Color, 1280, 720, 30, faceBitmapFormats); var rgbFrameReader = await rgbMediaCapture.Item1.CreateFrameReaderAsync(rgbMediaCapture.Item2); rgbFrameReader.AcquisitionMode = MediaFrameReaderAcquisitionMode.Realtime; int busyProcessingRgbFrame = 0; var unityWorldCoordinateSystem = Marshal.GetObjectForIUnknown(WorldManager.GetNativeISpatialCoordinateSystemPtr()) as SpatialCoordinateSystem; // Expecting this to run at 30fps. TypedEventHandler <MediaFrameReader, MediaFrameArrivedEventArgs> rgbFrameHandler = (sender, args) => { // Only proceed if we're not already 'busy' - i.e. we' if (Interlocked.CompareExchange(ref busyProcessingRgbFrame, 1, 0) == 0) { Task.Run( async() => { using (var rgbFrame = rgbFrameReader.TryAcquireLatestFrame()) { if ((rgbFrame != null) && (rgbFrame != lastRgbFrame)) { ++rgbProcessedCount; lastRgbFrame = rgbFrame; var facePosition = uVector3.zero; using (var videoFrame = rgbFrame.VideoMediaFrame.GetVideoFrame()) { var faces = await faceTracker.ProcessNextFrameAsync(videoFrame); var firstFace = faces.FirstOrDefault(); if (firstFace != null) { ++facesPresentCount; // Take the first face and the centre point of that face to try // and simplify things for my limited brain. var faceCentrePointInImageCoords = new Point( firstFace.FaceBox.X + (firstFace.FaceBox.Width / 2.0f), firstFace.FaceBox.Y + (firstFace.FaceBox.Height / 2.0f)); wMatrix4x4 projectionTransform = wMatrix4x4.Identity; wMatrix4x4 viewTransform = wMatrix4x4.Identity; wMatrix4x4 invertedViewTransform = wMatrix4x4.Identity; var rgbCoordinateSystem = GetRgbFrameProjectionAndCoordinateSystemDetails( rgbFrame, out projectionTransform, out invertedViewTransform); // Scale the RGB point (1280x720) var faceCentrePointUnitScaleRGB = ScalePointMinusOneToOne(faceCentrePointInImageCoords, rgbFrame); // Unproject the RGB point back at unit depth as per the locatable camera // document. var unprojectedFaceCentrePointRGB = UnProjectVector( new wVector3( (float)faceCentrePointUnitScaleRGB.X, (float)faceCentrePointUnitScaleRGB.Y, 1.0f), projectionTransform); // Transfrom this back by the inverted view matrix in order to put this into // the RGB camera coordinate system var faceCentrePointCameraCoordsRGB = wVector3.Transform(unprojectedFaceCentrePointRGB, invertedViewTransform); // Get the transform from the camera coordinate system to the Unity world // coordinate system, could probably cache this? var cameraRGBToWorldTransform = rgbCoordinateSystem.TryGetTransformTo(unityWorldCoordinateSystem); if (cameraRGBToWorldTransform.HasValue) { // Transform to world coordinates var faceCentrePointWorldCoords = wVector4.Transform( new wVector4( faceCentrePointCameraCoordsRGB.X, faceCentrePointCameraCoordsRGB.Y, faceCentrePointCameraCoordsRGB.Z, 1), cameraRGBToWorldTransform.Value); // Where's the camera in world coordinates? var cameraOriginWorldCoords = wVector4.Transform( new wVector4(0, 0, 0, 1), cameraRGBToWorldTransform.Value); // Multiply Z by -1 for Unity var cameraPoint = new uVector3( cameraOriginWorldCoords.X, cameraOriginWorldCoords.Y, -1.0f * cameraOriginWorldCoords.Z); // Multiply Z by -1 for Unity var facePoint = new uVector3( faceCentrePointWorldCoords.X, faceCentrePointWorldCoords.Y, -1.0f * faceCentrePointWorldCoords.Z); facePosition = cameraPoint + (facePoint - cameraPoint).normalized * centrePointDepthInMetres; } } } if (facePosition != uVector3.zero) { UnityEngine.WSA.Application.InvokeOnAppThread( () => { this.faceMarker.transform.position = facePosition; }, false ); } } } Interlocked.Exchange(ref busyProcessingRgbFrame, 0); } ); } else { Interlocked.Increment(ref rgbDroppedCount); } // NB: this is a bit naughty as I am accessing these counters across a few threads so // accuracy might suffer here. UnityEngine.WSA.Application.InvokeOnAppThread( () => { this.textMesh.text = $"{depthFrameCount} depth,{rgbProcessedCount} rgb done, {rgbDroppedCount} rgb drop," + $"{facesPresentCount} faces, ({centrePointDepthInMetres:N2})"; }, false); }; depthFrameReader.FrameArrived += depthFrameHandler; rgbFrameReader.FrameArrived += rgbFrameHandler; await depthFrameReader.StartAsync(); await rgbFrameReader.StartAsync(); // Wait forever then dispose...just doing this to keep track of what needs disposing. await Task.Delay(-1); depthFrameReader.FrameArrived -= depthFrameHandler; rgbFrameReader.FrameArrived -= rgbFrameHandler; rgbFrameReader.Dispose(); depthFrameReader.Dispose(); rgbMediaCapture.Item1.Dispose(); depthMediaCapture.Item1.Dispose(); Marshal.ReleaseComObject(unityWorldCoordinateSystem); }