public Option <HandInputEvent> Update(SkeletonFrame sf, byte[] cf) { if (sf != null) { if (skeletonData == null || skeletonData.Length != sf.SkeletonArrayLength) { skeletonData = new Skeleton[sf.SkeletonArrayLength]; } sf.CopySkeletonDataTo(skeletonData); var trackedIndex = SkeletonUtil.FirstTrackedSkeletonIndex(skeletonData); if (trackedIndex >= 0) { var skeleton = skeletonData[trackedIndex]; var handRight = SkeletonUtil.GetJoint(skeleton, JointType.HandRight); var handLeft = SkeletonUtil.GetJoint(skeleton, JointType.HandLeft); var handRightPos = coordMapper.MapSkeletonPointToDepthPoint(handRight.Position, DepthImageFormat.Resolution640x480Fps30); var handLeftPos = coordMapper.MapSkeletonPointToDepthPoint(handLeft.Position, DepthImageFormat.Resolution640x480Fps30); return(new Some <HandInputEvent>(new HandInputEvent(handLeftPos, handRightPos))); } } return(new None <HandInputEvent>()); }
public TrackingResult Update(short[] depthFrame, byte[] cf, Skeleton skeleton) { InterestPoints.Clear(); ConvertColorImage(cf); if (!initialized) { buffer.Init(smallGray.Ptr); initialized = true; } buffer.ProcessFrame(smallGray.Ptr); var stipList = buffer.GetInterestPoints(); foreach (Object p in stipList) { InterestPoints.Add(new Point(((Point)p).X * ImageScale, ((Point)p).Y * ImageScale)); } float z = DefaultZDist; if (skeleton != null) { var rightHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight); if (rightHandJoint.TrackingState == JointTrackingState.Tracked) { var point = mapper.MapSkeletonPointToColorPoint(rightHandJoint.Position); InterestPoints.Add(new Point(point.X, point.Y)); z = rightHandJoint.Position.Z; } } HandRect = ComputeInitialRect(z); playerDetector.FilterPlayer(depthFrame, cf); var depthImage = playerDetector.DepthImage; CvInvoke.cvSmooth(depthImage.Ptr, SmoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5, 0, 0); FindBestBoundingBox(HandRect); var depthBBs = new List <Rectangle>(); depthBBs.Add(HandRect); var colorBBs = new List <Rectangle>(); colorBBs.Add(HandRect); return(new TrackingResult(new None <Vector3D>(), new None <Vector3D>(), SmoothedDepth, depthBBs, null, colorBBs)); }
/// <summary> /// Detects the position of the hand nearest to the skeleton joint of the hand. /// </summary> /// <param name="depthData"></param> /// <returns></returns> public TrackingResult Update(short[] depthData, byte[] colorPixelData, Skeleton skeleton) { if (skeleton != null) { var skeHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight); var playerMask = CreatePlayerImage(depthData); if (colorPixelData != null) { SkinMask = skinDetetor.DetectSkin(colorPixelData); AlignColorImage(depthData, SkinMask); CvInvoke.cvAnd(playerMask.Ptr, alignedImg.Ptr, HandMask.Ptr, IntPtr.Zero); CvInvoke.cvDilate(HandMask.Ptr, HandMask.Ptr, Rect5, MorphIter); CvInvoke.cvDilate(HandMask.Ptr, HandMask.Ptr, IntPtr.Zero, MorphIter); } FindContours(skeHandJoint); RankCandidates(HandCandidates, skeHandJoint, PrevHand, depthData); if (HandCandidates.Count > 0) { var shoulderCenterJoint = SkeletonUtil.GetJoint(skeleton, JointType.ShoulderCenter); var detectSkeHandJointPos = FindHand(depthData, HandCandidates.First()); var relPos = SkeletonUtil.Sub(detectSkeHandJointPos, shoulderCenterJoint.Position); var angle = SkeletonUtil.PointDirection(detectSkeHandJointPos, SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position); var depthBBs = new List <Rectangle>(); depthBBs.Add(new Rectangle((int)(HandBox.center.X - HandBox.size.Width / 2), (int)(HandBox.center.Y - HandBox.size.Height / 2), (int)(HandBox.size.Width), (int)HandBox.size.Height)); return(new TrackingResult(new Some <Vector3D>(relPos), new Some <Vector3D>(angle), HandImage, depthBBs)); } } return(new TrackingResult()); }
/// <summary> /// /// </summary> /// <param name="depthFrame"></param> /// <param name="cf"></param> /// <param name="skeleton"></param> /// <returns>If skeleton is null, returns an empty result.</returns> public TrackingResult Update(short[] depthFrame, byte[] cf, Skeleton skeleton) { if (skeleton != null && depthFrame != null && cf != null) { float z = DefaultZDist; var rightHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight); var rightHandDepthPos = mapper.MapSkeletonPointToDepthPoint(rightHandJoint.Position); z = rightHandJoint.Position.Z; // Relatively rough estimate. InitialHandRect = ComputeInitialRect(rightHandDepthPos, z); if (!InitialHandRect.IsEmpty) { playerDetector.UpdateMasks(depthFrame, cf, InitialHandRect, true, true); var depthImage = playerDetector.DepthImage; CvInvoke.cvSmooth(depthImage.Ptr, SmoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5, 0, 0); HandRect = FindBestBoundingBox(InitialHandRect); if (!HandRect.IsEmpty) { var handSkeletonPoint = SkeletonUtil.DepthToSkeleton(HandRect, SmoothedDepth.Data, width, height, mapper); var relPos = SkeletonUtil.RelativePosToShoulder(handSkeletonPoint, skeleton); var angle = SkeletonUtil.PointDirection(handSkeletonPoint, SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position); var depthBBs = new List <Rectangle>(); var colorBBs = new List <Rectangle>(); depthBBs.Add(HandRect); colorBBs.Add(InitialHandRect); return(new TrackingResult(new Some <Vector3D>(relPos), new Some <Vector3D>(angle), SmoothedDepth, depthBBs, playerDetector.SkinImage, colorBBs)); } } } return(new TrackingResult()); }
/// <summary> /// If no bounding box is found, returns the last bounding box. /// </summary> /// <returns></returns> List <Rectangle> FindBestBoundingBox(short[] depthFrame, Skeleton skeleton) { CvInvoke.cvConvert(SaliencyProb.Ptr, TempMask.Ptr); // Non-zero pixels are treated as 1s. Source image content is modifield. CvInvoke.cvFindContours(TempMask.Ptr, storage, ref contourPtr, StructSize.MCvContour, RETR_TYPE.CV_RETR_EXTERNAL, CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, new Point(0, 0)); var contour = new Seq <Point>(contourPtr, null); SortedList <float, Seq <Point> > bestContours = new SortedList <float, Seq <Point> >(); List <Rectangle> bestBoundingBoxes = new List <Rectangle>(); float z = DefaultZDist; var hand = SkeletonUtil.GetJoint(skeleton, JointType.HandRight); if (hand != null) { z = hand.Position.Z; } double perimThresh = DepthUtil.GetDepthImageLength(width, HandWidth, z) * 2; FaceModel = SkeletonUtil.GetFaceModel(skeleton, mapper); for (; contour != null && contour.Ptr.ToInt32() != 0; contour = contour.HNext) { var perim = CvInvoke.cvContourPerimeter(contour.Ptr); if (perim > perimThresh) { var rect = contour.BoundingRectangle; var score = ContourScore(rect); var center = rect.Center(); int x = (int)center.X; int y = (int)center.Y; var depth = DepthUtil.RawToDepth(depthFrame[y * width + x]); if (!FaceModel.IsPartOfFace(x, y, depth) && (bestContours.Count < NumTrackedHands || score > bestContours.ElementAt(0).Key)) { bestContours.Add(score, contour); if (bestContours.Count > NumTrackedHands) { bestContours.RemoveAt(0); } } } } if (bestContours.Count > 0) { foreach (var c in bestContours.Values) { var rect = c.BoundingRectangle; CvInvoke.cvCamShift(TrackedDepthFrame.Ptr, rect, new MCvTermCriteria(CamShiftIter), out connectedComp, out shiftedBox); var bestBoundingBox = shiftedBox.MinAreaRect(); bestBoundingBoxes.Add(bestBoundingBox); //FloodFill(TrackedDepthFrame, bestBoundingBox); //if (bestBoundingBox.Width > 0) { // TempMask.ROI = bestBoundingBox; // CvInvoke.cvFindContours(TempMask.Ptr, storage, ref contourPtr, StructSize.MCvContour, // RETR_TYPE.CV_RETR_EXTERNAL, CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, // new Point(0, 0)); // contour = new Seq<Point>(contourPtr, null); // Seq<Point> largestContour = null; // var maxPerim = perimThresh; // for (; contour != null && contour.Ptr.ToInt32() != 0; contour = contour.HNext) { // var perim = CvInvoke.cvContourPerimeter(contour.Ptr); // if (perim > maxPerim) { // maxPerim = perim; // largestContour = contour; // } // } // CvInvoke.cvZero(TempMask.Ptr); // if (largestContour != null) // TempMask.Draw(largestContour, new Gray(255), -1); // FilterImage(TrackedDepthFrame, TempMask); // TempMask.ROI = Rectangle.Empty; //} } } return(bestBoundingBoxes); }
/// <summary> /// /// </summary> /// <param name="detphFrame">raw detph data.</param> /// <param name="skeleton">skeleton object from Kinect SDK. The skeleton coordinate space has /// the x-axis pointing rightwards, the y-axis pointing upwards, and the z-axis poining outwards /// relative to the image.</param> /// <returns>The position of the best bounding box relative to the shoulder joint in skeleton /// coordinates if the bounding box is valid. Otherwise returns None.</returns> public TrackingResult Update(short[] depthFrame, byte[] colorFrame, Skeleton skeleton) { t++; Option <Vector3D> relPos = new None <Vector3D>(); Option <Vector3D> angle = new None <Vector3D>(); if (skeleton == null || depthFrame == null) { return(new TrackingResult()); } playerDetector.FilterPlayerContourSkin(depthFrame, colorFrame); var depthImage = playerDetector.DepthImage; smoothedDepth.CopyTo(prevSmoothedDepth); // Median smoothing cannot be in place. CvInvoke.cvSmooth(depthImage.Ptr, smoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5, 0, 0); if (t > 1) { CvInvoke.cvAbsDiff(smoothedDepth.Ptr, prevSmoothedDepth.Ptr, Diff0.Ptr); //CvInvoke.cvErode(Diff0.Ptr, Diff0.Ptr, StructuringElement.Ptr, 1); DiffMask0.CopyTo(DiffMask1); CvInvoke.cvThreshold(Diff0.Ptr, DiffMask0.Ptr, 2, 255, THRESH.CV_THRESH_BINARY); if (t > 2) { // Makes diffMask1 the motion mask at t - 1. CvInvoke.cvAnd(DiffMask0.Ptr, DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero); if (bufferSize <= 1) { // Makes diffMask1 the motion mask at t - 0. CvInvoke.cvXor(DiffMask0.Ptr, DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero); } CvInvoke.cvMorphologyEx(DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero, IntPtr.Zero, CV_MORPH_OP.CV_MOP_OPEN, 1); ComputeCumulativeDist(Diff0, diffCumulativeDist); ComputeCumulativeDist(TrackedDepthFrame, depthCumulativeDist); CvInvoke.cvZero(SaliencyProb); var diffMaskData = DiffMask1.Data; var diffData = Diff0.Data; var depthData = TrackedDepthFrame.Data; var probData = SaliencyProb.Data; for (int i = 0; i < DiffMask1.Height; i++) { for (int j = 0; j < DiffMask1.Width; j++) { if (diffMaskData[i, j, 0] > 0) { var diffBin = diffData[i, j, 0]; var depthBin = depthData[i, j, 0]; probData[i, j, 0] = diffCumulativeDist[diffBin] * depthCumulativeDist[depthBin]; } } } PrevBoundingBoxes = FindBestBoundingBox(depthFrame, skeleton); if (PrevBoundingBoxes.LastOrDefault().Width > 0) { var handSkeletonPoint = SkeletonUtil.DepthToSkeleton(PrevBoundingBoxes.Last(), TrackedDepthFrame.Data, width, height, mapper); relPos = new Some <Vector3D>(SkeletonUtil.RelativePosToShoulder(handSkeletonPoint, skeleton)); angle = new Some <Vector3D>(SkeletonUtil.PointDirection(handSkeletonPoint, SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position)); } } } List <Rectangle> colorBBs = new List <Rectangle>(); foreach (var bb in PrevBoundingBoxes) { var colorBox = mapper.MapDepthRectToColorRect(bb, depthFrame, width, height); playerDetector.SmoothSkin(colorBox); colorBBs.Add(colorBox); } return(new TrackingResult(relPos, angle, TrackedDepthFrame, PrevBoundingBoxes, playerDetector.SkinImage, colorBBs)); }