private void ColorFrameReader_FrameArrived_FrameRenderer(MediaFrameReader sender, MediaFrameArrivedEventArgs args) { _frameRenderer.ProcessFrame(sender.TryAcquireLatestFrame()); }
private void FrameReader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args) { if (!frameProcessingSemaphore.Wait(0)) { return; } try { var frame = sender.TryAcquireLatestFrame(); if (frame != null) { this.frames[frame.SourceKind] = frame; } if (this.frames[MediaFrameSourceKind.Color] != null && this.frames[MediaFrameSourceKind.Depth] != null) { var colorDesc = this.frames[MediaFrameSourceKind.Color].VideoMediaFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read).GetPlaneDescription(0); var depthDesc = this.frames[MediaFrameSourceKind.Depth].VideoMediaFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read).GetPlaneDescription(0); // get points in 3d space DepthCorrelatedCoordinateMapper coordinateMapper = this.frames[MediaFrameSourceKind.Depth].VideoMediaFrame.DepthMediaFrame.TryCreateCoordinateMapper( this.frames[MediaFrameSourceKind.Color].VideoMediaFrame.CameraIntrinsics, this.frames[MediaFrameSourceKind.Color].CoordinateSystem); // get color information var bitmap = SoftwareBitmap.Convert(this.frames[MediaFrameSourceKind.Color].VideoMediaFrame.SoftwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Ignore); byte[] colorBytes = new byte[bitmap.PixelWidth * bitmap.PixelHeight * 4]; bitmap.CopyToBuffer(colorBytes.AsBuffer()); Vector3[] depthPoints = new Vector3[this.mapPoints.Length]; coordinateMapper.UnprojectPoints(this.mapPoints, this.frames[MediaFrameSourceKind.Color].CoordinateSystem, depthPoints); // resize image 1920x1080 -> 480x270 and apply depth filter byte[] resizedGrayColorBytes = new byte[240 * 135]; int row = 0; int at = 0; for (int idx = 0; idx < resizedGrayColorBytes.Length; ++idx) { var depth = depthPoints[idx]; // get depth bound for pixel idx float depthBound = 10.0f; if (float.IsNaN(depth.X)) { depthBound = 0.0f; } else { for (int i = 0; i < this.depthMap.Count; ++i) { if (this.depthMap[i].xMin < depth.X && depth.X < this.depthMap[i].xMax) { depthBound = this.depthMap[i].F(depth.X); break; } } } // get color of pixel idx // topLeft : at; topRight : at + strideX - 4; // bottomLeft : at + rgbWidth * (strideY - 1); bottomRight : at + rgbWidth * (strideY - 1) + strideX - 4; float bgr = 255; if (depth.Z < depthBound) { bgr = (Convert.ToInt16(colorBytes[at] + colorBytes[at + 28] + colorBytes[at + 53760] + colorBytes[at + 53788] + colorBytes[at + 1] + colorBytes[at + 29] + colorBytes[at + 53761] + colorBytes[at + 53789] + colorBytes[at + 2] + colorBytes[at + 30] + colorBytes[at + 53762] + colorBytes[at + 53790])) * 0.0833333f; } resizedGrayColorBytes[idx] = BitConverter.GetBytes(Convert.ToInt32(bgr))[0]; // iterate at += 32; if (at - row * 7680 > 7648) // at - row * rgbWidth > rgbWidth - strideX { row += 8; at = row * 7680; } } var faces = this.cascadeClassifier.DetectMultiScale(resizedGrayColorBytes, 240, 135); // debug image (optional) //byte[] dbg = new byte[240 * 135 + 4]; //Buffer.BlockCopy(BitConverter.GetBytes(240), 0, dbg, 0, 2); //Buffer.BlockCopy(BitConverter.GetBytes(135), 0, dbg, 2, 2); //Buffer.BlockCopy(resizedGrayColorBytes, 0, dbg, 4, 32400); //this.client.Publish("/kinect/face/debug", dbg); // reset face found status foreach (var log in this.faceLog) { log.SetFoundFalse(); } // create byte array from each face uint totalSize = 0; List <byte[]> faceBytesList = new List <byte[]>(); at = 1; int numDetectFaces = faces[0]; for (int j = 0; j < numDetectFaces; ++j) { // parse result from C++ uint xj = Convert.ToUInt32(faces[at++]) * 8; uint yj = Convert.ToUInt32(faces[at++]) * 8; uint width = Convert.ToUInt32(faces[at++]) * 8; // result is head + shoulder -> multiply 0.6 to get head only region uint height = Convert.ToUInt32(Convert.ToInt32(faces[at++]) * 8 * 0.6); // center crop image xj += Convert.ToUInt32(width * 0.2); width = Convert.ToUInt32(width * 0.6); uint size = width * height * 3 + 20; byte[] faceBytes = new byte[size]; totalSize += size; // get face 3d position var centerPoint = new Point(xj + Convert.ToUInt32(width * 0.5), yj + Convert.ToUInt32(height * 0.5)); var positionVector = coordinateMapper.UnprojectPoint(centerPoint, this.frames[MediaFrameSourceKind.Color].CoordinateSystem); // get likely face id from face position int likelyEnum = -1; float likeliness = float.MaxValue; for (int i = 0; i < this.faceLog.Count; ++i) { if (this.faceLog[i].isTracked && !this.faceLog[i].foundInThisFrame) { var dist = Vector3.Distance(positionVector, this.faceLog[i].facePosition); if (dist < 0.3 && dist < likeliness) // it is unlikely for a face to jump 30cm between frames { likelyEnum = i; likeliness = dist; } } } if (likelyEnum < 0) // if no likely face was found { for (int i = 0; i < this.faceLog.Count; ++i) { if (!this.faceLog[i].isTracked) // a new track is registered (will switch to isTracked when called Update) { likelyEnum = i; break; } } } if (likelyEnum < 0) // trackable number of faces already occupied, cannot track new face { continue; // id will be free once existing track is lost } this.faceLog[likelyEnum].Update(positionVector); // first 4 bytes is size of face image Array.Copy(BitConverter.GetBytes(width), 0, faceBytes, 0, 2); Array.Copy(BitConverter.GetBytes(height), 0, faceBytes, 2, 2); // next 12 bytes is 3d position of face var position = new float[] { positionVector.X, positionVector.Y, positionVector.Z }; Buffer.BlockCopy(position, 0, faceBytes, 4, 12); // next 4 bytes is face id Buffer.BlockCopy(BitConverter.GetBytes(this.faceLog[likelyEnum].id), 0, faceBytes, 16, 4); // copy rgb image for (int y = 0; y < height; ++y) { var srcIdx = Convert.ToInt32(((y + yj) * colorDesc.Width + xj) * 4); var destIdx = Convert.ToInt32((y * width) * 3 + 20); for (int x = 0; x < width; ++x) { Buffer.BlockCopy(colorBytes, srcIdx + x * 4, faceBytes, destIdx + x * 3, 3); } } faceBytesList.Add(faceBytes); } // for faces that were not found in current frame, release track state foreach (var log in this.faceLog) { if (log.isTracked && !log.foundInThisFrame) { ++log.lostTrackCount; // get fps, note, clock is always running when frame is being captured int fps = Convert.ToInt32(kinectFrameCount / this.appClock.Elapsed.TotalSeconds); if (log.lostTrackCount > 2 * fps) // lost for two seconds { log.Free(this.nextReservedFaceId); ++this.nextReservedFaceId; } } } // concatenate byte arrays to send (post-processed as totalSize not known in first foreach) int head = 1; // first 1 byte is number of faces byte[] bytes = new byte[totalSize + 1]; Array.Copy(BitConverter.GetBytes(faceBytesList.Count), 0, bytes, 0, head); foreach (byte[] faceByte in faceBytesList) { Array.Copy(faceByte, 0, bytes, head, faceByte.Length); head += faceByte.Length; } this.client.Publish("/kinect/detected/face", bytes); ++this.kinectFrameCount; bitmap.Dispose(); coordinateMapper.Dispose(); this.frames[MediaFrameSourceKind.Color].Dispose(); this.frames[MediaFrameSourceKind.Depth].Dispose(); this.frames[MediaFrameSourceKind.Color] = null; this.frames[MediaFrameSourceKind.Depth] = null; } } catch (Exception ex) { // TODO } finally { frameProcessingSemaphore.Release(); } }
private async void ActionButton2_Click(object sender, RoutedEventArgs e) { var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync(); // Color, infrared, and depth var selectedGroupObjects = frameSourceGroups.Select(group => new { sourceGroup = group, colorSourceInfo = group.SourceInfos.FirstOrDefault((sourceInfo) => { return(sourceInfo.SourceKind == MediaFrameSourceKind.Color); }) }).Where(t => t.colorSourceInfo != null) .FirstOrDefault(); MediaFrameSourceGroup selectedGroup = selectedGroupObjects?.sourceGroup; MediaFrameSourceInfo colorSourceInfo = selectedGroupObjects?.colorSourceInfo; if (selectedGroup == null) { return; } mediaCapture = new MediaCapture(); var settings = new MediaCaptureInitializationSettings() { SourceGroup = selectedGroup, SharingMode = MediaCaptureSharingMode.ExclusiveControl, MemoryPreference = MediaCaptureMemoryPreference.Cpu, StreamingCaptureMode = StreamingCaptureMode.Video }; try { await mediaCapture.InitializeAsync(settings); } catch (Exception ex) { System.Diagnostics.Debug.WriteLine("MediaCapture initialization failed: " + ex.Message); return; } var colorFrameSource = mediaCapture.FrameSources[colorSourceInfo.Id]; var preferredFormat = colorFrameSource.SupportedFormats.Where(format => { return(format.VideoFormat.Width == 1920); }).FirstOrDefault(); if (preferredFormat == null) { // Our desired format is not supported return; } await colorFrameSource.SetFormatAsync(preferredFormat); mediaFrameReader = await mediaCapture.CreateFrameReaderAsync(colorFrameSource, MediaEncodingSubtypes.Argb32); mediaFrameReader.FrameArrived += ColorFrameReader_FrameArrived_FrameRenderer; _frameRenderer = new FrameRenderer(imageElement); await mediaFrameReader.StartAsync(); }
private FrameGrabber(MediaCapture mediaCapture = null, MediaFrameSource mediaFrameSource = null, MediaFrameReader mediaFrameReader = null) { this.mediaCapture = mediaCapture; // capture audio, video , and image from camera this.mediaFrameSource = mediaFrameSource; // source of camera fames (color camera in this case) this.mediaFrameReader = mediaFrameReader; // access to frames from a MediaFrameSource then notifies when new frame arrives if (this.mediaFrameReader != null) { this.mediaFrameReader.FrameArrived += MediaFrameReader_FrameArrived; } }
private void FrameReader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args) { if (!frameProcessingSemaphore.Wait(0)) { return; } try { var frame = sender.TryAcquireLatestFrame(); if (frame != null) { this.frames[frame.SourceKind] = frame; } if (this.frames[MediaFrameSourceKind.Color] != null && this.frames[MediaFrameSourceKind.Depth] != null) { var colorDesc = this.frames[MediaFrameSourceKind.Color].VideoMediaFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read).GetPlaneDescription(0); var depthDesc = this.frames[MediaFrameSourceKind.Depth].VideoMediaFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read).GetPlaneDescription(0); // get points in 3d space DepthCorrelatedCoordinateMapper coordinateMapper = this.frames[MediaFrameSourceKind.Depth].VideoMediaFrame.DepthMediaFrame.TryCreateCoordinateMapper( this.frames[MediaFrameSourceKind.Color].VideoMediaFrame.CameraIntrinsics, this.frames[MediaFrameSourceKind.Color].CoordinateSystem); // get color information var bitmap = SoftwareBitmap.Convert(this.frames[MediaFrameSourceKind.Color].VideoMediaFrame.SoftwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Ignore); byte[] colorBytes = new byte[bitmap.PixelWidth * bitmap.PixelHeight * 4]; bitmap.CopyToBuffer(colorBytes.AsBuffer()); Vector3[] depthPoints = new Vector3[this.mapPoints.Length]; coordinateMapper.UnprojectPoints(this.mapPoints, this.frames[MediaFrameSourceKind.Color].CoordinateSystem, depthPoints); // resize image 1920x1080 -> 480x270 and apply depth filter byte[] resizedGrayColorBytes = new byte[240 * 135]; int row = 0; int at = 0; for (int idx = 0; idx < resizedGrayColorBytes.Length; ++idx) { var depth = depthPoints[idx]; // get depth bound for pixel idx float depthBound = 10.0f; if (float.IsNaN(depth.X)) { depthBound = 0.0f; } else { for (int i = 0; i < this.depthMap.Count; ++i) { if (this.depthMap[i].xMin < depth.X && depth.X < this.depthMap[i].xMax) { depthBound = this.depthMap[i].F(depth.X); break; } } } // get color of pixel idx // topLeft : at; topRight : at + strideX - 4; // bottomLeft : at + rgbWidth * (strideY - 1); bottomRight : at + rgbWidth * (strideY - 1) + strideX - 4; float bgr = 255; if (depth.Z < depthBound) { bgr = (Convert.ToInt16(colorBytes[at] + colorBytes[at + 28] + colorBytes[at + 53760] + colorBytes[at + 53788] + colorBytes[at + 1] + colorBytes[at + 29] + colorBytes[at + 53761] + colorBytes[at + 53789] + colorBytes[at + 2] + colorBytes[at + 30] + colorBytes[at + 53762] + colorBytes[at + 53790])) * 0.0833333f; } resizedGrayColorBytes[idx] = BitConverter.GetBytes(Convert.ToInt32(bgr))[0]; // iterate at += 32; if (at - row * 7680 > 7648) // at - row * rgbWidth > rgbWidth - strideX { row += 8; at = row * 7680; } } var faces = this.cascadeClassifier.DetectMultiScale(resizedGrayColorBytes, 240, 135); // debug image (optional) //byte[] dbg = new byte[240 * 135 + 4]; //Buffer.BlockCopy(BitConverter.GetBytes(240), 0, dbg, 0, 2); //Buffer.BlockCopy(BitConverter.GetBytes(135), 0, dbg, 2, 2); //Buffer.BlockCopy(resizedGrayColorBytes, 0, dbg, 4, 32400); //this.client.Publish("/kinect/face/debug", dbg); #if TRACKING_ON // reset face found status foreach (var log in this.faceLog) { log.SetFoundFalse(); } // parse data from cascadeClassifier List <Vector3> facePositionList = new List <Vector3>(); List <Tuple <uint, uint, uint, uint> > faceBoundsList = new List <Tuple <uint, uint, uint, uint> >(); at = 1; int numDetectFaces = faces[0]; for (int j = 0; j < numDetectFaces; ++j) { // parse result from C++ uint xj = Convert.ToUInt32(faces[at++]) * 8; uint yj = Convert.ToUInt32(faces[at++]) * 8; uint width = Convert.ToUInt32(faces[at++]) * 8; // result is head + shoulder -> multiply 0.6 to get head only region uint height = Convert.ToUInt32(Convert.ToInt32(faces[at++]) * 8 * 0.6); // center crop image xj += Convert.ToUInt32(width * 0.2); width = Convert.ToUInt32(width * 0.6); // get face 3d position var centerPoint = new Point(xj + Convert.ToUInt32(width * 0.5), yj + Convert.ToUInt32(height * 0.5)); var positionVector = coordinateMapper.UnprojectPoint(centerPoint, this.frames[MediaFrameSourceKind.Color].CoordinateSystem); facePositionList.Add(positionVector); faceBoundsList.Add(new Tuple <uint, uint, uint, uint>(xj, yj, width, height)); } // find face in current frame that matches log (only one face is linked to each log) int[] faceCorrespondingLog = Enumerable.Repeat(-1, facePositionList.Count).ToArray(); float[] faceCorrespondenceScore = Enumerable.Repeat(float.MaxValue, facePositionList.Count).ToArray(); for (int i = 0; i < this.faceLog.Count; ++i) { if (!faceLog[i].isTracked) { continue; // log is currently not used } int likelyEnum = -1; float likeliness = float.MaxValue; for (int j = 0; j < facePositionList.Count; ++j) { var dist = Vector3.Distance(facePositionList[j], this.faceLog[i].facePosition); if (dist < 0.3 && dist < likeliness) // it is unlikely for a face to jump 30cm between frames { likelyEnum = j; likeliness = dist; } } if (likelyEnum >= 0 && likeliness < faceCorrespondenceScore[likelyEnum]) { if (faceCorrespondingLog[likelyEnum] >= 0) { this.faceLog[faceCorrespondingLog[likelyEnum]].foundInThisFrame = false; // last log was not right match, undo match } faceCorrespondingLog[likelyEnum] = i; this.faceLog[i].foundInThisFrame = true; // this log is now matched with face } } // check faceCorrespondingLog and create byte array from each face uint totalSize = 0; List <byte[]> faceBytesList = new List <byte[]>(); for (int j = 0; j < faceCorrespondingLog.Length; ++j) { int likelyEnum = -1; Vector3 positionVector = facePositionList[j]; if (faceCorrespondingLog[j] < 0) // corresponding log was not yet found // find likely face log from logs { float likeliness = float.MaxValue; for (int i = 0; i < this.faceLog.Count; ++i) { if (this.faceLog[i].isTracked) { var dist = Vector3.Distance(positionVector, this.faceLog[i].facePosition); if (dist < 0.3 && dist < likeliness) // it is unlikely for a face to jump 30cm between frames { likelyEnum = i; likeliness = dist; } } } if (likelyEnum >= 0 && this.faceLog[likelyEnum].foundInThisFrame) { continue; // detected face was somehow a duplicate of an existing region, ignore } if (likelyEnum < 0) // if no likely face was found { for (int i = 0; i < this.faceLog.Count; ++i) { if (!this.faceLog[i].isTracked) // a new track is registered (will switch to isTracked when called Update) { likelyEnum = i; break; } } } if (likelyEnum < 0) // trackable number of faces already occupied, cannot track new face { continue; // id will be free once existing track is lost } } else // corresponding log is already found { likelyEnum = faceCorrespondingLog[j]; } this.faceLog[likelyEnum].Update(positionVector); uint xj = faceBoundsList[j].Item1; uint yj = faceBoundsList[j].Item2; uint width = faceBoundsList[j].Item3; uint height = faceBoundsList[j].Item4; uint size = width * height * 3 + 20; byte[] faceBytes = new byte[size]; totalSize += size; // first 4 bytes is size of face image Array.Copy(BitConverter.GetBytes(width), 0, faceBytes, 0, 2); Array.Copy(BitConverter.GetBytes(height), 0, faceBytes, 2, 2); // next 12 bytes is 3d position of face var position = new float[] { positionVector.X, positionVector.Y, positionVector.Z }; Buffer.BlockCopy(position, 0, faceBytes, 4, 12); // next 4 bytes is face id Buffer.BlockCopy(BitConverter.GetBytes(this.faceLog[likelyEnum].id), 0, faceBytes, 16, 4); // copy rgb image for (int y = 0; y < height; ++y) { var srcIdx = Convert.ToInt32(((y + yj) * colorDesc.Width + xj) * 4); var destIdx = Convert.ToInt32((y * width) * 3 + 20); for (int x = 0; x < width; ++x) { Buffer.BlockCopy(colorBytes, srcIdx + x * 4, faceBytes, destIdx + x * 3, 3); } } faceBytesList.Add(faceBytes); } // for faces that were not found in current frame, release track state foreach (var log in this.faceLog) { if (log.isTracked && !log.foundInThisFrame) { ++log.lostTrackCount; // get fps, note, clock is always running when frame is being captured int fps = Convert.ToInt32(kinectFrameCount / this.appClock.Elapsed.TotalSeconds); if (log.lostTrackCount > 10 * fps) // lost for ten seconds { log.Free(this.nextReservedFaceId); ++this.nextReservedFaceId; } } } #else // parse data from cascadeClassifier uint totalSize = 0; List <byte[]> faceBytesList = new List <byte[]>(); at = 1; int numDetectFaces = faces[0]; for (int j = 0; j < numDetectFaces; ++j) { #if VALID_FACE_CHECK // parse result from C++ uint xjRaw = Convert.ToUInt32(faces[at++]); uint yjRaw = Convert.ToUInt32(faces[at++]); uint widthRaw = Convert.ToUInt32(faces[at++]); // result is head + shoulder -> multiply 0.6 to get head only region uint heightRaw = Convert.ToUInt32(Convert.ToInt32(faces[at++]) * 0.6); // center crop image xjRaw += Convert.ToUInt32(widthRaw * 0.2); widthRaw = Convert.ToUInt32(widthRaw * 0.6); uint xj = xjRaw * 8; uint yj = yjRaw * 8; uint width = widthRaw * 8; uint height = heightRaw * 8; #else // parse result from C++ uint xj = Convert.ToUInt32(faces[at++]) * 8; uint yj = Convert.ToUInt32(faces[at++]) * 8; uint width = Convert.ToUInt32(faces[at++]) * 8; // result is head + shoulder -> multiply 0.6 to get head only region uint height = Convert.ToUInt32(Convert.ToInt32(faces[at++]) * 8 * 0.6); // center crop image xj += Convert.ToUInt32(width * 0.2); width = Convert.ToUInt32(width * 0.6); #endif // get face 3d position var centerPoint = new Point(xj + Convert.ToUInt32(width * 0.5), yj + Convert.ToUInt32(height * 0.5)); var positionVector = coordinateMapper.UnprojectPoint(centerPoint, this.frames[MediaFrameSourceKind.Color].CoordinateSystem); #if VALID_FACE_CHECK // find y height uint yjMax = yjRaw + heightRaw; uint xjMax = xjRaw + widthRaw; float threshold_z = positionVector.Z + 0.25f; float minimumY = float.MaxValue; float maximumY = float.MinValue; for (uint y = yjRaw; y < yjMax; ++y) { int rowPoints = 0; float averageY = 0.0f; uint point = xjRaw + y * 240; var pxy = depthPoints[point]; for (uint x = xjRaw; x < xjMax; ++x) { if (!float.IsNaN(pxy.Y) && !float.IsInfinity(pxy.Y) && (pxy.Z < threshold_z)) { averageY += pxy.Y; ++rowPoints; } pxy = depthPoints[++point]; } averageY /= rowPoints; if (rowPoints != 0) { if (averageY < minimumY) { minimumY = averageY; } else if (averageY > maximumY) { maximumY = averageY; } } } if ((maximumY - minimumY) > 0.35 || (maximumY - minimumY) < 0.1) // unlikely a face { continue; } #endif uint size = width * height * 3 + 20; byte[] faceBytes = new byte[size]; totalSize += size; // first 4 bytes is size of face image Array.Copy(BitConverter.GetBytes(width), 0, faceBytes, 0, 2); Array.Copy(BitConverter.GetBytes(height), 0, faceBytes, 2, 2); // next 12 bytes is 3d position of face var position = new float[] { positionVector.X, positionVector.Y, positionVector.Z }; Buffer.BlockCopy(position, 0, faceBytes, 4, 12); // next 4 bytes is face id (dummy) Buffer.BlockCopy(BitConverter.GetBytes(j), 0, faceBytes, 16, 4); // copy rgb image for (int y = 0; y < height; ++y) { var srcIdx = Convert.ToInt32(((y + yj) * colorDesc.Width + xj) * 4); var destIdx = Convert.ToInt32((y * width) * 3 + 20); for (int x = 0; x < width; ++x) { Buffer.BlockCopy(colorBytes, srcIdx + x * 4, faceBytes, destIdx + x * 3, 3); } } faceBytesList.Add(faceBytes); } #endif // concatenate byte arrays to send (post-processed as totalSize not known in first foreach) int head = 1; // first 1 byte is number of faces byte[] bytes = new byte[totalSize + 1]; Array.Copy(BitConverter.GetBytes(faceBytesList.Count), 0, bytes, 0, head); foreach (byte[] faceByte in faceBytesList) { Array.Copy(faceByte, 0, bytes, head, faceByte.Length); head += faceByte.Length; } this.client.Publish("/kinect/detected/face", bytes); ++this.kinectFrameCount; bitmap.Dispose(); coordinateMapper.Dispose(); this.frames[MediaFrameSourceKind.Color].Dispose(); this.frames[MediaFrameSourceKind.Depth].Dispose(); this.frames[MediaFrameSourceKind.Color] = null; this.frames[MediaFrameSourceKind.Depth] = null; } } catch (Exception ex) { // TODO } finally { frameProcessingSemaphore.Release(); } }
public async Task Initialize(VideoSetting videoSetting) { await DispatcherHelper.RunAndAwaitAsync(CoreApplication.MainView.CoreWindow.Dispatcher, CoreDispatcherPriority.Normal, async() => // await CoreApplication.MainView.CoreWindow.Dispatcher.RunAsyncRunAndAwaitAsync(CoreDispatcherPriority.Normal, async () => { _threadsCount = videoSetting.UsedThreads; _stoppedThreads = videoSetting.UsedThreads; _lastFrameAdded.Start(); _imageQuality = new BitmapPropertySet(); var imageQualityValue = new BitmapTypedValue(videoSetting.VideoQuality, Windows.Foundation.PropertyType.Single); _imageQuality.Add("ImageQuality", imageQualityValue); _mediaCapture = new MediaCapture(); var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync(); var settings = new MediaCaptureInitializationSettings() { SharingMode = MediaCaptureSharingMode.ExclusiveControl, //With CPU the results contain always SoftwareBitmaps, otherwise with GPU //they preferring D3DSurface MemoryPreference = MediaCaptureMemoryPreference.Cpu, //Capture only video, no audio StreamingCaptureMode = StreamingCaptureMode.Video }; await _mediaCapture.InitializeAsync(settings); var mediaFrameSource = _mediaCapture.FrameSources.First().Value; var videoDeviceController = mediaFrameSource.Controller.VideoDeviceController; videoDeviceController.DesiredOptimization = Windows.Media.Devices.MediaCaptureOptimization.Quality; videoDeviceController.PrimaryUse = Windows.Media.Devices.CaptureUse.Video; //Set exposure (auto light adjustment) if (_mediaCapture.VideoDeviceController.Exposure.Capabilities.Supported && _mediaCapture.VideoDeviceController.Exposure.Capabilities.AutoModeSupported) { _mediaCapture.VideoDeviceController.Exposure.TrySetAuto(true); } var videoResolutionWidthHeight = VideoResolutionWidthHeight.Get(videoSetting.VideoResolution); var videoSubType = VideoSubtypeHelper.Get(videoSetting.VideoSubtype); //Set resolution, frame rate and video subtyp var videoFormat = mediaFrameSource.SupportedFormats.Where(sf => sf.VideoFormat.Width == videoResolutionWidthHeight.Width && sf.VideoFormat.Height == videoResolutionWidthHeight.Height && sf.Subtype == videoSubType) .OrderByDescending(m => m.FrameRate.Numerator / m.FrameRate.Denominator) .First(); await mediaFrameSource.SetFormatAsync(videoFormat); _mediaFrameReader = await _mediaCapture.CreateFrameReaderAsync(mediaFrameSource); await _mediaFrameReader.StartAsync(); }); }
/// <summary> /// Stops reading from the frame reader, disposes of the reader and updates the button state. /// </summary> private async Task StopReaderAsync() { _streaming = false; if (_reader != null) { await _reader.StopAsync(); _reader.FrameArrived -= Reader_FrameArrived; _reader.Dispose(); _reader = null; _logger.Log("Reader stopped."); } await UpdateButtonStateAsync(); }
public async Task Initialize() { await CoreApplication.MainView.CoreWindow.Dispatcher.RunAndAwaitAsync(CoreDispatcherPriority.Normal, async() => { _imageQuality = new BitmapPropertySet(); var imageQualityValue = new BitmapTypedValue(IMAGE_QUALITY_PERCENT, Windows.Foundation.PropertyType.Single); _imageQuality.Add("ImageQuality", imageQualityValue); _mediaCapture = new MediaCapture(); var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync(); var settings = new MediaCaptureInitializationSettings() { SharingMode = MediaCaptureSharingMode.ExclusiveControl, //With CPU the results contain always SoftwareBitmaps, otherwise with GPU //they preferring D3DSurface MemoryPreference = MediaCaptureMemoryPreference.Cpu, //Capture only video, no audio StreamingCaptureMode = StreamingCaptureMode.Video }; await _mediaCapture.InitializeAsync(settings); var mediaFrameSource = _mediaCapture.FrameSources.First().Value; var videoDeviceController = mediaFrameSource.Controller.VideoDeviceController; videoDeviceController.DesiredOptimization = Windows.Media.Devices.MediaCaptureOptimization.Quality; videoDeviceController.PrimaryUse = Windows.Media.Devices.CaptureUse.Video; //Set backlight compensation to min (otherwise there are problems with strong light sources) if (videoDeviceController.BacklightCompensation.Capabilities.Supported) { videoDeviceController.BacklightCompensation.TrySetValue(videoDeviceController.BacklightCompensation.Capabilities.Min); } //Set exposure (auto light adjustment) if (_mediaCapture.VideoDeviceController.Exposure.Capabilities.Supported && _mediaCapture.VideoDeviceController.Exposure.Capabilities.AutoModeSupported) { _mediaCapture.VideoDeviceController.Exposure.TrySetAuto(true); } //Set resolution, frame rate and video subtyp var videoFormat = mediaFrameSource.SupportedFormats.First(sf => sf.VideoFormat.Width == VIDEO_WIDTH && sf.VideoFormat.Height == VIDEO_HEIGHT && sf.Subtype == VIDEO_SUBTYP); await mediaFrameSource.SetFormatAsync(videoFormat); _mediaFrameReader = await _mediaCapture.CreateFrameReaderAsync(mediaFrameSource); //If debugger is attached you can't get frames from the camera, because the BitmapEncoder //has a bug and not dispose correctly. This results in an System.OutOfMemoryException if (!Debugger.IsAttached) { _mediaFrameReader.FrameArrived += FrameArrived; await _mediaFrameReader.StartAsync(); } }); }
internal BodyFrameReader(Sensor sensor, MediaFrameReader bodyReader) { Sensor = sensor; _bodyReader = bodyReader; }
/// <summary> /// The Task to asynchronously initialize MediaCapture in UWP. The camera of HoloLens will /// be configured to preview video of 896x504 at 30 fps, pixel format is NV12. MediaFrameReader /// will be initialized and register the callback function OnFrameArrived to each video /// frame. Note that this task does not start running the video preview, but configures the /// running behavior. This task should be executed when ARUWPController status is /// ARUWP_STATUS_CLEAN, and will change it to ARUWP_STATUS_VIDEO_INITIALIZED if no error /// occurred. [internal use] /// </summary> /// <returns>Whether video pipeline is successfully initialized</returns> public async Task <bool> InitializeMediaCaptureAsyncTask() { if (controller.status != ARUWP.ARUWP_STATUS_CLEAN) { Debug.Log(TAG + ": InitializeMediaCaptureAsyncTask() unsupported status"); return(false); } if (mediaCapture != null) { Debug.Log(TAG + ": InitializeMediaCaptureAsyncTask() fails because mediaCapture is not null"); return(false); } var allGroups = await MediaFrameSourceGroup.FindAllAsync(); foreach (var group in allGroups) { Debug.Log(group.DisplayName + ", " + group.Id); } if (allGroups.Count <= 0) { Debug.Log(TAG + ": InitializeMediaCaptureAsyncTask() fails because there is no MediaFrameSourceGroup"); return(false); } Config.SourceSelectionDictionary.TryGetValue(Config.Mono, out isPublishing); if (isPublishing) { rosConnector = controller.rosConnector; publisher = new RosSharp.RosBridgeClient.NonMono.Publisher <RosSharp.RosBridgeClient.Messages.Sensor.Image>(ref rosConnector, "/hololens/" + Config.Mono); FrameReadyToPublish += PublishFrame; } // Initialize mediacapture with the source group. mediaCapture = new MediaCapture(); var settings = new MediaCaptureInitializationSettings { SourceGroup = allGroups[0], // This media capture can share streaming with other apps. SharingMode = MediaCaptureSharingMode.SharedReadOnly, // Only stream video and don't initialize audio capture devices. StreamingCaptureMode = StreamingCaptureMode.Video, // Set to CPU to ensure frames always contain CPU SoftwareBitmap images // instead of preferring GPU D3DSurface images. MemoryPreference = MediaCaptureMemoryPreference.Cpu }; await mediaCapture.InitializeAsync(settings); Debug.Log(TAG + ": MediaCapture is successfully initialized in shared mode."); try { int targetVideoWidth, targetVideoHeight; float targetVideoFrameRate; switch (videoParameter) { case VideoParameter.Param1280x720x15: targetVideoWidth = 1280; targetVideoHeight = 720; targetVideoFrameRate = 15.0f; break; case VideoParameter.Param1280x720x30: targetVideoWidth = 1280; targetVideoHeight = 720; targetVideoFrameRate = 30.0f; break; case VideoParameter.Param1344x756x15: targetVideoWidth = 1344; targetVideoHeight = 756; targetVideoFrameRate = 15.0f; break; case VideoParameter.Param1344x756x30: targetVideoWidth = 1344; targetVideoHeight = 756; targetVideoFrameRate = 30.0f; break; case VideoParameter.Param896x504x15: targetVideoWidth = 896; targetVideoHeight = 504; targetVideoFrameRate = 15.0f; break; case VideoParameter.Param896x504x30: targetVideoWidth = 896; targetVideoHeight = 504; targetVideoFrameRate = 30.0f; break; default: targetVideoWidth = 896; targetVideoHeight = 504; targetVideoFrameRate = 30.0f; break; } var mediaFrameSourceVideoPreview = mediaCapture.FrameSources.Values.Single(x => x.Info.MediaStreamType == MediaStreamType.VideoPreview); MediaFrameFormat targetResFormat = null; float framerateDiffMin = 60f; foreach (var f in mediaFrameSourceVideoPreview.SupportedFormats.OrderBy(x => x.VideoFormat.Width * x.VideoFormat.Height)) { if (f.VideoFormat.Width == targetVideoWidth && f.VideoFormat.Height == targetVideoHeight) { if (targetResFormat == null) { targetResFormat = f; framerateDiffMin = Mathf.Abs(f.FrameRate.Numerator / f.FrameRate.Denominator - targetVideoFrameRate); } else if (Mathf.Abs(f.FrameRate.Numerator / f.FrameRate.Denominator - targetVideoFrameRate) < framerateDiffMin) { targetResFormat = f; framerateDiffMin = Mathf.Abs(f.FrameRate.Numerator / f.FrameRate.Denominator - targetVideoFrameRate); } } } if (targetResFormat == null) { targetResFormat = mediaFrameSourceVideoPreview.SupportedFormats[0]; Debug.Log(TAG + ": Unable to choose the selected format, fall back"); targetResFormat = mediaFrameSourceVideoPreview.SupportedFormats.OrderBy(x => x.VideoFormat.Width * x.VideoFormat.Height).FirstOrDefault(); } await mediaFrameSourceVideoPreview.SetFormatAsync(targetResFormat); frameReader = await mediaCapture.CreateFrameReaderAsync(mediaFrameSourceVideoPreview, targetResFormat.Subtype); frameReader.FrameArrived += OnFrameArrived; controller.frameWidth = Convert.ToInt32(targetResFormat.VideoFormat.Width); controller.frameHeight = Convert.ToInt32(targetResFormat.VideoFormat.Height); videoBufferSize = controller.frameWidth * controller.frameHeight * 4; Debug.Log(TAG + ": FrameReader is successfully initialized, " + controller.frameWidth + "x" + controller.frameHeight + ", Framerate: " + targetResFormat.FrameRate.Numerator + "/" + targetResFormat.FrameRate.Denominator); } catch (Exception e) { Debug.Log(TAG + ": FrameReader is not initialized"); Debug.Log(TAG + ": Exception: " + e); return(false); } if (isPublishing) { double fps; if (Config.FrameRateDictionary.TryGetValue(Config.Mono, out fps)) { publishPeriod = 1 / fps; var bytesPerPixel = (int)HololensRobotController.WindowsSensors.CameraHandler.GetBytesPerPixel(MediaFrameSourceKind.Color); int pixelBufferSize = (int)(controller.frameHeight * controller.frameWidth * bytesPerPixel); publishRowSize = controller.frameWidth * bytesPerPixel; publishBuffer = new byte[pixelBufferSize]; } else { isPublishing = false; } } controller.status = ARUWP.ARUWP_STATUS_VIDEO_INITIALIZED; signalInitDone = true; Debug.Log(TAG + ": InitializeMediaCaptureAsyncTask() is successful"); return(true); }
/// <summary> /// The callback that is triggered when new video preview frame arrives. In this function, /// video frame is saved for Unity UI if videoPreview is enabled, tracking task is triggered /// in this function call, and video FPS is recorded. [internal use] /// </summary> /// <param name="sender">MediaFrameReader object</param> /// <param name="args">arguments not used here</param> private void OnFrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args) { ARUWPUtils.VideoTick(); using (var frame = sender.TryAcquireLatestFrame()) { if (frame != null) { float[] cameraToWorldMatrixAsFloat; if (TryGetCameraToWorldMatrix(frame, out cameraToWorldMatrixAsFloat) == false) { return; } Matrix4x4 latestLocatableCameraToWorld = ConvertFloatArrayToMatrix4x4(cameraToWorldMatrixAsFloat); var originalSoftwareBitmap = frame.VideoMediaFrame.SoftwareBitmap; var softwareBitmap = SoftwareBitmap.Convert(originalSoftwareBitmap, BitmapPixelFormat.Rgba8, BitmapAlphaMode.Ignore); originalSoftwareBitmap?.Dispose(); if (videoPreview) { Interlocked.Exchange(ref _bitmap, softwareBitmap); controller.ProcessFrameAsync(SoftwareBitmap.Copy(softwareBitmap), latestLocatableCameraToWorld); } else { controller.ProcessFrameAsync(SoftwareBitmap.Copy(softwareBitmap), latestLocatableCameraToWorld); } signalTrackingUpdated = true; if (isPublishing) { TimeSpan sampledCurrentTime = HololensRobotController.Utilities.Timer.SampleCurrentStopwatch(); double elapsedTotalSeconds = HololensRobotController.Utilities.Timer.GetElapsedTimeInSeconds(sampledCurrentTime); if (elapsedTotalSeconds >= nextPublishTime) { SoftwareBitmap publishedBitmap; if (Config.convertColorToGrayscale) { publishedBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Gray8); originalSoftwareBitmap?.Dispose(); } else { publishedBitmap = softwareBitmap; } IBuffer ibuffer = publishBuffer.AsBuffer(); publishedBitmap.CopyToBuffer(ibuffer); OnFrameReadyToPublish( new HololensRobotController.WindowsSensors.FrameEventArgs( (uint)controller.frameHeight, (uint)controller.frameWidth, HololensRobotController.WindowsSensors.CameraHandler.GetCameraSourceEncoding(MediaFrameSourceKind.Color), (uint)publishRowSize, publishBuffer, sampledCurrentTime + HololensRobotController.Utilities.Timer.GetOffsetUTC())); nextPublishTime = nextPublishTime + publishPeriod; } } } } }
/// <summary> /// Event handler for video frames for the local video capture device. /// </summary> private async void FrameArrivedHandler(MediaFrameReader sender, MediaFrameArrivedEventArgs e) { if (!_isClosed) { if (!_videoFormatManager.SelectedFormat.IsEmpty() && (OnVideoSourceEncodedSample != null || OnVideoSourceRawSample != null)) { using (var mediaFrameReference = sender.TryAcquireLatestFrame()) { var videoMediaFrame = mediaFrameReference?.VideoMediaFrame; var softwareBitmap = videoMediaFrame?.SoftwareBitmap; if (softwareBitmap == null && videoMediaFrame != null) { var videoFrame = videoMediaFrame.GetVideoFrame(); softwareBitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(videoFrame.Direct3DSurface); } if (softwareBitmap != null) { int width = softwareBitmap.PixelWidth; int height = softwareBitmap.PixelHeight; if (softwareBitmap.BitmapPixelFormat != BitmapPixelFormat.Nv12) { softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Nv12, BitmapAlphaMode.Ignore); } // Swap the processed frame to _backBuffer and dispose of the unused image. softwareBitmap = Interlocked.Exchange(ref _backBuffer, softwareBitmap); using (BitmapBuffer buffer = _backBuffer.LockBuffer(BitmapBufferAccessMode.Read)) { using (var reference = buffer.CreateReference()) { unsafe { byte *dataInBytes; uint capacity; ((IMemoryBufferByteAccess)reference).GetBuffer(out dataInBytes, out capacity); byte[] nv12Buffer = new byte[capacity]; Marshal.Copy((IntPtr)dataInBytes, nv12Buffer, 0, (int)capacity); if (OnVideoSourceEncodedSample != null) { lock (_videoEncoder) { var encodedBuffer = _videoEncoder.EncodeVideo(width, height, nv12Buffer, EncoderInputFormat, _videoFormatManager.SelectedFormat.Codec); if (encodedBuffer != null) { uint fps = (_fpsDenominator > 0 && _fpsNumerator > 0) ? _fpsNumerator / _fpsDenominator : DEFAULT_FRAMES_PER_SECOND; uint durationRtpTS = VIDEO_SAMPLING_RATE / fps; OnVideoSourceEncodedSample.Invoke(durationRtpTS, encodedBuffer); } if (_forceKeyFrame) { _forceKeyFrame = false; } } } if (OnVideoSourceRawSample != null) { uint frameSpacing = 0; if (_lastFrameAt != DateTime.MinValue) { frameSpacing = Convert.ToUInt32(DateTime.Now.Subtract(_lastFrameAt).TotalMilliseconds); } var bgrBuffer = PixelConverter.NV12toBGR(nv12Buffer, width, height, width * 3); OnVideoSourceRawSample(frameSpacing, width, height, bgrBuffer, VideoPixelFormatsEnum.Bgr); } } } } _backBuffer?.Dispose(); softwareBitmap?.Dispose(); } _lastFrameAt = DateTime.Now; } } } }
/// <summary> /// Attempts to initialise the local video capture device. /// </summary> /// <param name="width">The frame width to attempt to initialise the video capture device with. Set as 0 to use default.</param> /// <param name="height">The frame height to attempt to initialise the video capture device with. Set as 0 to use default.</param> /// <param name="fps">The frame rate, in frames per second, to attempt to initialise the video capture device with. /// Set as 0 to use default.</param> private async Task <bool> InitialiseDevice(uint width, uint height, uint fps) { var mediaCaptureSettings = new MediaCaptureInitializationSettings() { StreamingCaptureMode = StreamingCaptureMode.Video, SharingMode = MediaCaptureSharingMode.ExclusiveControl, MediaCategory = MediaCategory.Communications }; if (_videoDeviceID != null) { var vidCapDevices = await DeviceInformation.FindAllAsync(DeviceClass.VideoCapture).AsTask().ConfigureAwait(false); var vidDevice = vidCapDevices.FirstOrDefault(x => x.Id == _videoDeviceID || x.Name == _videoDeviceID); if (vidDevice == null) { logger.LogWarning($"Could not find video capture device for specified ID {_videoDeviceID}, using default device."); } else { logger.LogInformation($"Video capture device {vidDevice.Name} selected."); mediaCaptureSettings.VideoDeviceId = vidDevice.Id; } } await _mediaCapture.InitializeAsync(mediaCaptureSettings).AsTask().ConfigureAwait(false); MediaFrameSourceInfo colorSourceInfo = null; foreach (var srcInfo in _mediaCapture.FrameSources) { if (srcInfo.Value.Info.MediaStreamType == MediaStreamType.VideoRecord && srcInfo.Value.Info.SourceKind == MediaFrameSourceKind.Color) { colorSourceInfo = srcInfo.Value.Info; break; } } var colorFrameSource = _mediaCapture.FrameSources[colorSourceInfo.Id]; var preferredFormat = colorFrameSource.SupportedFormats.Where(format => { return(format.VideoFormat.Width >= _width && format.VideoFormat.Width >= _height && (format.FrameRate.Numerator / format.FrameRate.Denominator) >= fps && format.Subtype == MF_NV12_PIXEL_FORMAT); }).FirstOrDefault(); if (preferredFormat == null) { // Try again without the pixel format. preferredFormat = colorFrameSource.SupportedFormats.Where(format => { return(format.VideoFormat.Width >= _width && format.VideoFormat.Width >= _height && (format.FrameRate.Numerator / format.FrameRate.Denominator) >= fps); }).FirstOrDefault(); } if (preferredFormat == null) { // Still can't get what we want. Log a warning message and take the default. logger.LogWarning($"The video capture device did not support the requested format (or better) {_width}x{_height} {fps}fps. Using default mode."); preferredFormat = colorFrameSource.SupportedFormats.First(); } if (preferredFormat == null) { throw new ApplicationException("The video capture device does not support a compatible video format for the requested parameters."); } await colorFrameSource.SetFormatAsync(preferredFormat).AsTask().ConfigureAwait(false); _mediaFrameReader = await _mediaCapture.CreateFrameReaderAsync(colorFrameSource).AsTask().ConfigureAwait(false); _mediaFrameReader.AcquisitionMode = MediaFrameReaderAcquisitionMode.Realtime; // Frame source and format have now been successfully set. _width = preferredFormat.VideoFormat.Width; _height = preferredFormat.VideoFormat.Height; _fpsNumerator = preferredFormat.FrameRate.Numerator; _fpsDenominator = preferredFormat.FrameRate.Denominator; //double fpsSelected = _fpsNumerator / _fpsDenominator; //string pixFmt = preferredFormat.Subtype == MF_I420_PIXEL_FORMAT ? "I420" : preferredFormat.Subtype; //string deviceName = colorFrameSource.Info.DeviceInformation.Name; //logger.LogInformation($"Video capture device {deviceName} successfully initialised: {_width}x{_height} {fpsSelected:0.##}fps pixel format {pixFmt}."); PrintFrameSourceInfo(colorFrameSource); _mediaFrameReader.FrameArrived += FrameArrivedHandler; return(true); }
public static async Task <FrameGrabber> CreateAsync() { MediaCapture mediaCapture = null; MediaFrameReader mediaFrameReader = null; MediaFrameSourceGroup selectedGroup = null; MediaFrameSourceInfo selectedSourceInfo = null; var groups = await MediaFrameSourceGroup.FindAllAsync(); foreach (MediaFrameSourceGroup sourceGroup in groups) { // there should be only one color source for the HoloLens foreach (MediaFrameSourceInfo sourceInfo in sourceGroup.SourceInfos) { if (sourceInfo.SourceKind == MediaFrameSourceKind.Color) { selectedSourceInfo = sourceInfo; break; } } if (selectedSourceInfo != null) { selectedGroup = sourceGroup; } } // define the type of MediaCapture we want (Initialize MediaCapture to capture video from a color camera on teh CPU) var settings = new MediaCaptureInitializationSettings { SourceGroup = selectedGroup, SharingMode = MediaCaptureSharingMode.SharedReadOnly, StreamingCaptureMode = StreamingCaptureMode.Video, MemoryPreference = MediaCaptureMemoryPreference.Cpu, }; mediaCapture = new MediaCapture(); try { await mediaCapture.InitializeAsync(settings); } catch (Exception e) { Debug.WriteLine($"Can't initialize MediaCapture {e.ToString()}"); return(new FrameGrabber()); } // if initialization is successful, obtain MediaFrameSource and create MediaFrameReader MediaFrameSource selectedSource = mediaCapture.FrameSources[selectedSourceInfo.Id]; mediaFrameReader = await mediaCapture.CreateFrameReaderAsync(selectedSource); // ensure MediaFrameReader is successfully created to instantiate Grabber instance MediaFrameReaderStartStatus status = await mediaFrameReader.StartAsync(); if (status == MediaFrameReaderStartStatus.Success) { return(new FrameGrabber(mediaCapture, selectedSource, mediaFrameReader)); } else { return(new FrameGrabber()); } }
private async void ActionButton_Click(object sender, RoutedEventArgs e) { // <SnippetImageElementSource> imageElement.Source = new SoftwareBitmapSource(); // </SnippetImageElementSource> // <SnippetFindAllAsync> var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync(); // </SnippetFindAllAsync> // Color, infrared, and depth // <SnippetSelectColor> var selectedGroupObjects = frameSourceGroups.Select(group => new { sourceGroup = group, colorSourceInfo = group.SourceInfos.FirstOrDefault((sourceInfo) => { // On XBox/Kinect, omit the MediaStreamType and EnclosureLocation tests return(sourceInfo.MediaStreamType == MediaStreamType.VideoPreview && sourceInfo.SourceKind == MediaFrameSourceKind.Color && sourceInfo.DeviceInformation?.EnclosureLocation.Panel == Windows.Devices.Enumeration.Panel.Front); }) }).Where(t => t.colorSourceInfo != null) .FirstOrDefault(); MediaFrameSourceGroup selectedGroup = selectedGroupObjects?.sourceGroup; MediaFrameSourceInfo colorSourceInfo = selectedGroupObjects?.colorSourceInfo; if (selectedGroup == null) { return; } // </SnippetSelectColor> // <SnippetInitMediaCapture> mediaCapture = new MediaCapture(); var settings = new MediaCaptureInitializationSettings() { SourceGroup = selectedGroup, SharingMode = MediaCaptureSharingMode.ExclusiveControl, MemoryPreference = MediaCaptureMemoryPreference.Cpu, StreamingCaptureMode = StreamingCaptureMode.Video }; try { await mediaCapture.InitializeAsync(settings); } catch (Exception ex) { System.Diagnostics.Debug.WriteLine("MediaCapture initialization failed: " + ex.Message); return; } // </SnippetInitMediaCapture> var colorFrameSource = mediaCapture.FrameSources[colorSourceInfo.Id]; var preferredFormat = colorFrameSource.SupportedFormats.Where(format => { return(format.VideoFormat.Width == 1920); }).FirstOrDefault(); if (preferredFormat == null) { // Our desired format is not supported return; } await colorFrameSource.SetFormatAsync(preferredFormat); // <SnippetCreateFrameReader> mediaFrameReader = await mediaCapture.CreateFrameReaderAsync(colorFrameSource, MediaEncodingSubtypes.Argb32); mediaFrameReader.FrameArrived += ColorFrameReader_FrameArrived; await mediaFrameReader.StartAsync(); // </SnippetCreateFrameReader> }
/// <summary> /// Initialize camera pipeline resources and register a callback for when new VideoFrames become available. /// </summary> /// <returns></returns> private async Task InitializeAsync() { // Initialize MediaCapture with default settings in video-only streaming mode. // We first try to aquire exclusive sharing mode and if we fail, we then attempt again in shared mode // so that multiple instances can access the camera concurrently m_mediaCapture = new MediaCapture(); var mediaCaptureInistializationSettings = new MediaCaptureInitializationSettings() { StreamingCaptureMode = StreamingCaptureMode.Video, SharingMode = m_sharingMode }; // Register a callback in case MediaCapture fails. This can happen for example if another app is using the camera and we can't get ExclusiveControl m_mediaCapture.Failed += MediaCapture_Failed; await m_mediaCapture.InitializeAsync(mediaCaptureInistializationSettings); // Get a list of available Frame source and iterate through them to find a video preview or // a video record source with color images (and not IR, depth or other types) var selectedFrameSource = m_mediaCapture.FrameSources.FirstOrDefault(source => source.Value.Info.MediaStreamType == MediaStreamType.VideoPreview && source.Value.Info.SourceKind == MediaFrameSourceKind.Color).Value; if (selectedFrameSource == null) { selectedFrameSource = m_mediaCapture.FrameSources.FirstOrDefault(source => source.Value.Info.MediaStreamType == MediaStreamType.VideoRecord && source.Value.Info.SourceKind == MediaFrameSourceKind.Color).Value; } if (selectedFrameSource == null) { throw new Exception("No valid video frame sources were found with source type color."); } Console.WriteLine($"{selectedFrameSource.Info.DeviceInformation?.Name} | MediaStreamType: {selectedFrameSource.Info.MediaStreamType} MediaFrameSourceKind: {selectedFrameSource.Info.SourceKind}"); // If initializing in ExclusiveControl mode, attempt to use a 15fps+ BGRA8 format natively from the camera. // If not, just use whatever format is already set. MediaFrameFormat selectedFormat = selectedFrameSource.CurrentFormat; if (m_sharingMode == MediaCaptureSharingMode.ExclusiveControl) { var mediaFrameFormats = selectedFrameSource.SupportedFormats.OrderByDescending((format) => format.VideoFormat.Width * format.VideoFormat.Height); selectedFormat = mediaFrameFormats.Where( format => format.FrameRate.Numerator / format.FrameRate.Denominator >= 15 && // fps string.Compare(format.Subtype, MediaEncodingSubtypes.Bgra8, true) == 0).FirstOrDefault(); // If not possible, then try to use other supported format at 15fps+ if (selectedFormat == null) { selectedFormat = mediaFrameFormats.Where( format => format.FrameRate.Numerator / format.FrameRate.Denominator >= 15 && // fps (string.Compare(format.Subtype, MediaEncodingSubtypes.Nv12, true) == 0 || string.Compare(format.Subtype, MediaEncodingSubtypes.Yuy2, true) == 0 || string.Compare(format.Subtype, MediaEncodingSubtypes.Rgb32, true) == 0)).FirstOrDefault(); } if (selectedFormat == null) { throw (new Exception("No suitable media format found on the selected source")); } await selectedFrameSource.SetFormatAsync(selectedFormat); selectedFormat = selectedFrameSource.CurrentFormat; Console.WriteLine($"Attempting to set camera source to {selectedFormat.Subtype} : " + $"{selectedFormat.VideoFormat.Width}x{selectedFormat.VideoFormat.Height}" + $"@{selectedFormat.FrameRate.Numerator / selectedFormat.FrameRate.Denominator}fps"); } Console.WriteLine($"Frame source format: {selectedFormat.Subtype} : " + $"{selectedFormat.VideoFormat.Width}x{selectedFormat.VideoFormat.Height}" + $"@{selectedFormat.FrameRate.Numerator / selectedFormat.FrameRate.Denominator}fps"); m_frameReader = await m_mediaCapture.CreateFrameReaderAsync(selectedFrameSource); m_frameReader.FrameArrived += FrameArrivedHandler; m_frameReader.AcquisitionMode = MediaFrameReaderAcquisitionMode.Realtime; await m_frameReader.StartAsync(); }
/// <summary> /// Start the video stream. This just prepares the stream for capture, and doesn't start collecting frames /// </summary> /// <param name="streamDesc">The description of the stream to start.</param> public async void Start(StreamDescription streamDesc) { #if CAN_USE_UWP_TYPES lock (stateLock) { if (State != CameraState.Initialized) { throw new InvalidOperationException("Start cannot be called until the camera is in the Initialized state"); } State = CameraState.Starting; } Resolution = streamDesc.Resolution; CameraType = streamDesc.CameraType; StreamDescriptionInternal desc = streamDesc as StreamDescriptionInternal; MediaCaptureInitializationSettings initSettings = new MediaCaptureInitializationSettings() { SourceGroup = desc.FrameSourceGroup, SharingMode = MediaCaptureSharingMode.ExclusiveControl, MemoryPreference = MediaCaptureMemoryPreference.Cpu, StreamingCaptureMode = StreamingCaptureMode.Video }; // initialize the media device mediaCapture = new MediaCapture(); try { await mediaCapture.InitializeAsync(initSettings); } catch (Exception ex) { System.Diagnostics.Debug.WriteLine($"MediaCapture initialization failed: {ex.Message}"); mediaCapture.Dispose(); mediaCapture = null; } if (mediaCapture != null) { // get access to the video device controller for property settings videoDeviceController = mediaCapture.VideoDeviceController; // choose media source MediaFrameSource frameSource = mediaCapture.FrameSources[desc.FrameSourceInfo.Id]; MediaFrameFormat preferredFormat = null; foreach (MediaFrameFormat format in frameSource.SupportedFormats) { if (format.VideoFormat.Width == desc.Resolution.Width && format.VideoFormat.Height == desc.Resolution.Height && Math.Abs((double)format.FrameRate.Numerator / (double)format.FrameRate.Denominator - desc.Resolution.Framerate) < epsilon) { preferredFormat = format; break; } } if (preferredFormat != null && preferredFormat != frameSource.CurrentFormat) { await frameSource.SetFormatAsync(preferredFormat); } else { System.Diagnostics.Debug.WriteLine($"failed to set desired frame format"); } // set up frame readercapture frame data frameReader = await mediaCapture.CreateFrameReaderAsync(frameSource); frameReader.FrameArrived += OnMediaFrameArrived; await frameReader.StartAsync(); lock (stateLock) { State = CameraState.Ready; OnCameraStarted?.Invoke(this, true); } } else { lock (stateLock) { // drop back to initialized when the camera doesn't initialize State = CameraState.Initialized; OnCameraStarted?.Invoke(this, false); } } #else await Task.CompletedTask; #endif }
internal BodyFrameReader(Sensor sensor, MediaFrameReader bodyReader) { Sensor = sensor; _bodyReader = bodyReader; _bodyReader.FrameArrived += BodyFrameReader_FrameArrived; }
/// <summary> /// Creates a frame reader from the current frame source and registers to handle its frame events. /// </summary> private async Task CreateReaderAsync() { await InitializeCaptureAsync(); UpdateFrameSource(); if (_source != null) { _reader = await _mediaCapture.CreateFrameReaderAsync(_source); _reader.FrameArrived += Reader_FrameArrived; _logger.Log($"Reader created on source: {_source.Info.Id}"); } }
internal void Dispose() { _bodyReader?.Dispose(); _bodyReader = null; }
/// <summary> /// Handles the frame arrived event by converting the frame to a displayable /// format and rendering it to the screen. /// </summary> private void Reader_FrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args) { // TryAcquireLatestFrame will return the latest frame that has not yet been acquired. // This can return null if there is no such frame, or if the reader is not in the // "Started" state. The latter can occur if a FrameArrived event was in flight // when the reader was stopped. using (var frame = sender.TryAcquireLatestFrame()) { _frameRenderer.ProcessFrame(frame); } }
public async Task StartAsync(string Name, bool UseGpu = false) { var frameSourceGroups = await AsAsync(MediaFrameSourceGroup.FindAllAsync()); var selectedGroup = frameSourceGroups.Where(x => x.DisplayName.Contains(Name)).FirstOrDefault(); if (null == selectedGroup) { throw new ApplicationException($"Unable to find frame source named {Name}"); } var colorSourceInfo = selectedGroup.SourceInfos .Where(x => x.MediaStreamType == MediaStreamType.VideoRecord && x.SourceKind == MediaFrameSourceKind.Color) .FirstOrDefault(); if (null == colorSourceInfo) { throw new ApplicationException($"Unable to find color video recording source on {Name}"); } mediaCapture = new MediaCapture(); if (null == mediaCapture) { throw new ApplicationException($"Unable to create new mediacapture"); } var settings = new MediaCaptureInitializationSettings() { SourceGroup = selectedGroup, SharingMode = MediaCaptureSharingMode.ExclusiveControl, MemoryPreference = UseGpu ? MediaCaptureMemoryPreference.Auto : MediaCaptureMemoryPreference.Cpu, StreamingCaptureMode = StreamingCaptureMode.Video }; try { await AsAsync(mediaCapture.InitializeAsync(settings)); } catch (Exception ex) { throw new ApplicationException("MediaCapture initialization failed: " + ex.Message, ex); } var colorFrameSource = mediaCapture.FrameSources[colorSourceInfo.Id]; var preferredFormat = colorFrameSource.SupportedFormats.Where(format => format.VideoFormat.Width >= 1080 && format.Subtype == "NV12").FirstOrDefault(); if (null == preferredFormat) { throw new ApplicationException("Our desired format is not supported"); } await AsAsync(colorFrameSource.SetFormatAsync(preferredFormat)); mediaFrameReader = await AsAsync(mediaCapture.CreateFrameReaderAsync(colorFrameSource, MediaEncodingSubtypes.Argb32)); if (null == mediaFrameReader) { throw new ApplicationException($"Unable to create new mediaframereader"); } evtFrame = new EventWaitHandle(false, EventResetMode.ManualReset); mediaFrameReader.FrameArrived += (s, a) => evtFrame.Set(); await AsAsync(mediaFrameReader.StartAsync()); Log.WriteLineVerbose("FrameReader Started"); }