/// <summary>
/// Detects the position of the hand nearest to the skeleton joint of the hand.
/// </summary>
/// <param name="depthData"></param>
/// <returns></returns>
public TrackingResult Update(short[] depthData, byte[] colorPixelData, Skeleton skeleton)
{
if (skeleton != null)
{
var skeHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
var playerMask = CreatePlayerImage(depthData);
if (colorPixelData != null)
{
SkinMask = skinDetetor.DetectSkin(colorPixelData);
AlignColorImage(depthData, SkinMask);
CvInvoke.cvAnd(playerMask.Ptr, alignedImg.Ptr, HandMask.Ptr, IntPtr.Zero);
CvInvoke.cvDilate(HandMask.Ptr, HandMask.Ptr, Rect5, MorphIter);
CvInvoke.cvDilate(HandMask.Ptr, HandMask.Ptr, IntPtr.Zero, MorphIter);
}
FindContours(skeHandJoint);
RankCandidates(HandCandidates, skeHandJoint, PrevHand, depthData);
if (HandCandidates.Count > 0)
{
var shoulderCenterJoint = SkeletonUtil.GetJoint(skeleton, JointType.ShoulderCenter);
var detectSkeHandJointPos = FindHand(depthData, HandCandidates.First());
var relPos = SkeletonUtil.Sub(detectSkeHandJointPos, shoulderCenterJoint.Position);
var angle = SkeletonUtil.PointDirection(detectSkeHandJointPos,
SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position);
var depthBBs = new List <Rectangle>();
depthBBs.Add(new Rectangle((int)(HandBox.center.X - HandBox.size.Width / 2),
(int)(HandBox.center.Y - HandBox.size.Height / 2), (int)(HandBox.size.Width),
(int)HandBox.size.Height));
return(new TrackingResult(new Some <Vector3D>(relPos),
new Some <Vector3D>(angle), HandImage, depthBBs));
}
}
return(new TrackingResult());
}
/// <summary>
///
/// </summary>
/// <param name="depthFrame"></param>
/// <param name="cf"></param>
/// <param name="skeleton"></param>
/// <returns>If skeleton is null, returns an empty result.</returns>
public TrackingResult Update(short[] depthFrame, byte[] cf, Skeleton skeleton)
{
if (skeleton != null && depthFrame != null && cf != null)
{
float z = DefaultZDist;
var rightHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
var rightHandDepthPos = mapper.MapSkeletonPointToDepthPoint(rightHandJoint.Position);
z = rightHandJoint.Position.Z;
// Relatively rough estimate.
InitialHandRect = ComputeInitialRect(rightHandDepthPos, z);
if (!InitialHandRect.IsEmpty)
{
playerDetector.UpdateMasks(depthFrame, cf, InitialHandRect, true, true);
var depthImage = playerDetector.DepthImage;
CvInvoke.cvSmooth(depthImage.Ptr, SmoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5, 0, 0);
HandRect = FindBestBoundingBox(InitialHandRect);
if (!HandRect.IsEmpty)
{
var handSkeletonPoint = SkeletonUtil.DepthToSkeleton(HandRect, SmoothedDepth.Data,
width, height, mapper);
var relPos = SkeletonUtil.RelativePosToShoulder(handSkeletonPoint, skeleton);
var angle = SkeletonUtil.PointDirection(handSkeletonPoint,
SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position);
var depthBBs = new List <Rectangle>();
var colorBBs = new List <Rectangle>();
depthBBs.Add(HandRect);
colorBBs.Add(InitialHandRect);
return(new TrackingResult(new Some <Vector3D>(relPos), new Some <Vector3D>(angle),
SmoothedDepth, depthBBs, playerDetector.SkinImage, colorBBs));
}
}
}
return(new TrackingResult());
}
/// <summary>
///
/// </summary>
/// <param name="detphFrame">raw detph data.</param>
/// <param name="skeleton">skeleton object from Kinect SDK. The skeleton coordinate space has
/// the x-axis pointing rightwards, the y-axis pointing upwards, and the z-axis poining outwards
/// relative to the image.</param>
/// <returns>The position of the best bounding box relative to the shoulder joint in skeleton
/// coordinates if the bounding box is valid. Otherwise returns None.</returns>
public TrackingResult Update(short[] depthFrame, byte[] colorFrame, Skeleton skeleton)
{
t++;
Option <Vector3D> relPos = new None <Vector3D>();
Option <Vector3D> angle = new None <Vector3D>();
if (skeleton == null || depthFrame == null)
{
return(new TrackingResult());
}
playerDetector.FilterPlayerContourSkin(depthFrame, colorFrame);
var depthImage = playerDetector.DepthImage;
smoothedDepth.CopyTo(prevSmoothedDepth);
// Median smoothing cannot be in place.
CvInvoke.cvSmooth(depthImage.Ptr, smoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5,
0, 0);
if (t > 1)
{
CvInvoke.cvAbsDiff(smoothedDepth.Ptr, prevSmoothedDepth.Ptr, Diff0.Ptr);
//CvInvoke.cvErode(Diff0.Ptr, Diff0.Ptr, StructuringElement.Ptr, 1);
DiffMask0.CopyTo(DiffMask1);
CvInvoke.cvThreshold(Diff0.Ptr, DiffMask0.Ptr, 2, 255, THRESH.CV_THRESH_BINARY);
if (t > 2)
{
// Makes diffMask1 the motion mask at t - 1.
CvInvoke.cvAnd(DiffMask0.Ptr, DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero);
if (bufferSize <= 1)
{
// Makes diffMask1 the motion mask at t - 0.
CvInvoke.cvXor(DiffMask0.Ptr, DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero);
}
CvInvoke.cvMorphologyEx(DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero, IntPtr.Zero,
CV_MORPH_OP.CV_MOP_OPEN, 1);
ComputeCumulativeDist(Diff0, diffCumulativeDist);
ComputeCumulativeDist(TrackedDepthFrame, depthCumulativeDist);
CvInvoke.cvZero(SaliencyProb);
var diffMaskData = DiffMask1.Data;
var diffData = Diff0.Data;
var depthData = TrackedDepthFrame.Data;
var probData = SaliencyProb.Data;
for (int i = 0; i < DiffMask1.Height; i++)
{
for (int j = 0; j < DiffMask1.Width; j++)
{
if (diffMaskData[i, j, 0] > 0)
{
var diffBin = diffData[i, j, 0];
var depthBin = depthData[i, j, 0];
probData[i, j, 0] = diffCumulativeDist[diffBin] * depthCumulativeDist[depthBin];
}
}
}
PrevBoundingBoxes = FindBestBoundingBox(depthFrame, skeleton);
if (PrevBoundingBoxes.LastOrDefault().Width > 0)
{
var handSkeletonPoint = SkeletonUtil.DepthToSkeleton(PrevBoundingBoxes.Last(),
TrackedDepthFrame.Data, width, height, mapper);
relPos = new Some <Vector3D>(SkeletonUtil.RelativePosToShoulder(handSkeletonPoint,
skeleton));
angle = new Some <Vector3D>(SkeletonUtil.PointDirection(handSkeletonPoint,
SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position));
}
}
}
List <Rectangle> colorBBs = new List <Rectangle>();
foreach (var bb in PrevBoundingBoxes)
{
var colorBox = mapper.MapDepthRectToColorRect(bb, depthFrame, width, height);
playerDetector.SmoothSkin(colorBox);
colorBBs.Add(colorBox);
}
return(new TrackingResult(relPos, angle, TrackedDepthFrame, PrevBoundingBoxes,
playerDetector.SkinImage, colorBBs));
}