void ReplayFrame(ReplayDepthImageFrame df, ReplayColorImageFrame cf,
ReplaySkeletonFrame sf)
{
if (df != null)
{
labelFrameNumber.Content = df.FrameNumber.ToString();
if (gtReplayer != null)
{
var data = gtReplayer.GetDataFrame(df.FrameNumber);
if (data != null)
{
UpdateGroundTruthDisplay(data);
}
}
}
colorManager.Update(cf, !displayDebug);
depthManager.Update(df);
UpdateSkeletonDisplay(sf);
if (handTracker != null && recogEngine != null)
{
var result = handTracker.Update(depthManager.PixelData, colorManager.PixelData,
SkeletonUtil.FirstTrackedSkeleton(sf.Skeletons));
var gesture = recogEngine.Update(result, viewHog);
lock (inputServer)
inputServer.Send(gesture);
textGestureEvent.Text = gesture;
fpsCounter.LogFPS();
UpdateDisplay(result);
}
}
public Option <HandInputEvent> Update(SkeletonFrame sf, byte[] cf)
{
if (sf != null)
{
if (skeletonData == null || skeletonData.Length != sf.SkeletonArrayLength)
{
skeletonData = new Skeleton[sf.SkeletonArrayLength];
}
sf.CopySkeletonDataTo(skeletonData);
var trackedIndex = SkeletonUtil.FirstTrackedSkeletonIndex(skeletonData);
if (trackedIndex >= 0)
{
var skeleton = skeletonData[trackedIndex];
var handRight = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
var handLeft = SkeletonUtil.GetJoint(skeleton, JointType.HandLeft);
var handRightPos = coordMapper.MapSkeletonPointToDepthPoint(handRight.Position,
DepthImageFormat.Resolution640x480Fps30);
var handLeftPos = coordMapper.MapSkeletonPointToDepthPoint(handLeft.Position,
DepthImageFormat.Resolution640x480Fps30);
return(new Some <HandInputEvent>(new HandInputEvent(handLeftPos, handRightPos)));
}
}
return(new None <HandInputEvent>());
}
public TrackingResult Update(short[] depthFrame, byte[] cf, Skeleton skeleton)
{
InterestPoints.Clear();
ConvertColorImage(cf);
if (!initialized)
{
buffer.Init(smallGray.Ptr);
initialized = true;
}
buffer.ProcessFrame(smallGray.Ptr);
var stipList = buffer.GetInterestPoints();
foreach (Object p in stipList)
{
InterestPoints.Add(new Point(((Point)p).X * ImageScale, ((Point)p).Y * ImageScale));
}
float z = DefaultZDist;
if (skeleton != null)
{
var rightHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
if (rightHandJoint.TrackingState == JointTrackingState.Tracked)
{
var point = mapper.MapSkeletonPointToColorPoint(rightHandJoint.Position);
InterestPoints.Add(new Point(point.X, point.Y));
z = rightHandJoint.Position.Z;
}
}
HandRect = ComputeInitialRect(z);
playerDetector.FilterPlayer(depthFrame, cf);
var depthImage = playerDetector.DepthImage;
CvInvoke.cvSmooth(depthImage.Ptr, SmoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5, 0, 0);
FindBestBoundingBox(HandRect);
var depthBBs = new List <Rectangle>();
depthBBs.Add(HandRect);
var colorBBs = new List <Rectangle>();
colorBBs.Add(HandRect);
return(new TrackingResult(new None <Vector3D>(), new None <Vector3D>(), SmoothedDepth,
depthBBs, null, colorBBs));
}
void kinectRuntime_AllFrameReady(object sender, AllFramesReadyEventArgs e)
{
// If replaying, bypass this.
if (replay != null && !replay.IsFinished)
{
return;
}
using (var cf = e.OpenColorImageFrame())
using (var df = e.OpenDepthImageFrame())
using (var sf = e.OpenSkeletonFrame()) {
try {
if (recorder != null && sf != null && df != null && cf != null)
{
recorder.Record(sf, df, cf);
}
} catch (ObjectDisposedException) { }
if (cf != null)
{
colorManager.Update(cf, !displayDebug);
}
if (df != null)
{
depthFrameNumber = df.FrameNumber;
depthManager.Update(df);
}
if (sf != null)
{
UpdateSkeletonDisplay(sf);
if (handTracker != null && recogEngine != null)
{
var result = handTracker.Update(depthManager.PixelData, colorManager.PixelData,
SkeletonUtil.FirstTrackedSkeleton(sf.GetSkeletons()));
var gesture = recogEngine.Update(result);
lock (inputServer)
inputServer.Send(gesture);
UpdateDisplay(result);
textGestureEvent.Text = gesture;
fpsCounter.LogFPS();
}
}
}
}
/// <summary>
/// Detects the position of the hand nearest to the skeleton joint of the hand.
/// </summary>
/// <param name="depthData"></param>
/// <returns></returns>
public TrackingResult Update(short[] depthData, byte[] colorPixelData, Skeleton skeleton)
{
if (skeleton != null)
{
var skeHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
var playerMask = CreatePlayerImage(depthData);
if (colorPixelData != null)
{
SkinMask = skinDetetor.DetectSkin(colorPixelData);
AlignColorImage(depthData, SkinMask);
CvInvoke.cvAnd(playerMask.Ptr, alignedImg.Ptr, HandMask.Ptr, IntPtr.Zero);
CvInvoke.cvDilate(HandMask.Ptr, HandMask.Ptr, Rect5, MorphIter);
CvInvoke.cvDilate(HandMask.Ptr, HandMask.Ptr, IntPtr.Zero, MorphIter);
}
FindContours(skeHandJoint);
RankCandidates(HandCandidates, skeHandJoint, PrevHand, depthData);
if (HandCandidates.Count > 0)
{
var shoulderCenterJoint = SkeletonUtil.GetJoint(skeleton, JointType.ShoulderCenter);
var detectSkeHandJointPos = FindHand(depthData, HandCandidates.First());
var relPos = SkeletonUtil.Sub(detectSkeHandJointPos, shoulderCenterJoint.Position);
var angle = SkeletonUtil.PointDirection(detectSkeHandJointPos,
SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position);
var depthBBs = new List <Rectangle>();
depthBBs.Add(new Rectangle((int)(HandBox.center.X - HandBox.size.Width / 2),
(int)(HandBox.center.Y - HandBox.size.Height / 2), (int)(HandBox.size.Width),
(int)HandBox.size.Height));
return(new TrackingResult(new Some <Vector3D>(relPos),
new Some <Vector3D>(angle), HandImage, depthBBs));
}
}
return(new TrackingResult());
}
/// <summary>
///
/// </summary>
/// <param name="depthFrame"></param>
/// <param name="cf"></param>
/// <param name="skeleton"></param>
/// <returns>If skeleton is null, returns an empty result.</returns>
public TrackingResult Update(short[] depthFrame, byte[] cf, Skeleton skeleton)
{
if (skeleton != null && depthFrame != null && cf != null)
{
float z = DefaultZDist;
var rightHandJoint = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
var rightHandDepthPos = mapper.MapSkeletonPointToDepthPoint(rightHandJoint.Position);
z = rightHandJoint.Position.Z;
// Relatively rough estimate.
InitialHandRect = ComputeInitialRect(rightHandDepthPos, z);
if (!InitialHandRect.IsEmpty)
{
playerDetector.UpdateMasks(depthFrame, cf, InitialHandRect, true, true);
var depthImage = playerDetector.DepthImage;
CvInvoke.cvSmooth(depthImage.Ptr, SmoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5, 0, 0);
HandRect = FindBestBoundingBox(InitialHandRect);
if (!HandRect.IsEmpty)
{
var handSkeletonPoint = SkeletonUtil.DepthToSkeleton(HandRect, SmoothedDepth.Data,
width, height, mapper);
var relPos = SkeletonUtil.RelativePosToShoulder(handSkeletonPoint, skeleton);
var angle = SkeletonUtil.PointDirection(handSkeletonPoint,
SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position);
var depthBBs = new List <Rectangle>();
var colorBBs = new List <Rectangle>();
depthBBs.Add(HandRect);
colorBBs.Add(InitialHandRect);
return(new TrackingResult(new Some <Vector3D>(relPos), new Some <Vector3D>(angle),
SmoothedDepth, depthBBs, playerDetector.SkinImage, colorBBs));
}
}
}
return(new TrackingResult());
}
/// <summary>
/// If no bounding box is found, returns the last bounding box.
/// </summary>
/// <returns></returns>
List <Rectangle> FindBestBoundingBox(short[] depthFrame, Skeleton skeleton)
{
CvInvoke.cvConvert(SaliencyProb.Ptr, TempMask.Ptr);
// Non-zero pixels are treated as 1s. Source image content is modifield.
CvInvoke.cvFindContours(TempMask.Ptr, storage, ref contourPtr, StructSize.MCvContour,
RETR_TYPE.CV_RETR_EXTERNAL, CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, new Point(0, 0));
var contour = new Seq <Point>(contourPtr, null);
SortedList <float, Seq <Point> > bestContours = new SortedList <float, Seq <Point> >();
List <Rectangle> bestBoundingBoxes = new List <Rectangle>();
float z = DefaultZDist;
var hand = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
if (hand != null)
{
z = hand.Position.Z;
}
double perimThresh = DepthUtil.GetDepthImageLength(width, HandWidth, z) * 2;
FaceModel = SkeletonUtil.GetFaceModel(skeleton, mapper);
for (; contour != null && contour.Ptr.ToInt32() != 0; contour = contour.HNext)
{
var perim = CvInvoke.cvContourPerimeter(contour.Ptr);
if (perim > perimThresh)
{
var rect = contour.BoundingRectangle;
var score = ContourScore(rect);
var center = rect.Center();
int x = (int)center.X;
int y = (int)center.Y;
var depth = DepthUtil.RawToDepth(depthFrame[y * width + x]);
if (!FaceModel.IsPartOfFace(x, y, depth) &&
(bestContours.Count < NumTrackedHands || score > bestContours.ElementAt(0).Key))
{
bestContours.Add(score, contour);
if (bestContours.Count > NumTrackedHands)
{
bestContours.RemoveAt(0);
}
}
}
}
if (bestContours.Count > 0)
{
foreach (var c in bestContours.Values)
{
var rect = c.BoundingRectangle;
CvInvoke.cvCamShift(TrackedDepthFrame.Ptr, rect, new MCvTermCriteria(CamShiftIter),
out connectedComp, out shiftedBox);
var bestBoundingBox = shiftedBox.MinAreaRect();
bestBoundingBoxes.Add(bestBoundingBox);
//FloodFill(TrackedDepthFrame, bestBoundingBox);
//if (bestBoundingBox.Width > 0) {
// TempMask.ROI = bestBoundingBox;
// CvInvoke.cvFindContours(TempMask.Ptr, storage, ref contourPtr, StructSize.MCvContour,
// RETR_TYPE.CV_RETR_EXTERNAL, CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE,
// new Point(0, 0));
// contour = new Seq<Point>(contourPtr, null);
// Seq<Point> largestContour = null;
// var maxPerim = perimThresh;
// for (; contour != null && contour.Ptr.ToInt32() != 0; contour = contour.HNext) {
// var perim = CvInvoke.cvContourPerimeter(contour.Ptr);
// if (perim > maxPerim) {
// maxPerim = perim;
// largestContour = contour;
// }
// }
// CvInvoke.cvZero(TempMask.Ptr);
// if (largestContour != null)
// TempMask.Draw(largestContour, new Gray(255), -1);
// FilterImage(TrackedDepthFrame, TempMask);
// TempMask.ROI = Rectangle.Empty;
//}
}
}
return(bestBoundingBoxes);
}
/// <summary>
///
/// </summary>
/// <param name="detphFrame">raw detph data.</param>
/// <param name="skeleton">skeleton object from Kinect SDK. The skeleton coordinate space has
/// the x-axis pointing rightwards, the y-axis pointing upwards, and the z-axis poining outwards
/// relative to the image.</param>
/// <returns>The position of the best bounding box relative to the shoulder joint in skeleton
/// coordinates if the bounding box is valid. Otherwise returns None.</returns>
public TrackingResult Update(short[] depthFrame, byte[] colorFrame, Skeleton skeleton)
{
t++;
Option <Vector3D> relPos = new None <Vector3D>();
Option <Vector3D> angle = new None <Vector3D>();
if (skeleton == null || depthFrame == null)
{
return(new TrackingResult());
}
playerDetector.FilterPlayerContourSkin(depthFrame, colorFrame);
var depthImage = playerDetector.DepthImage;
smoothedDepth.CopyTo(prevSmoothedDepth);
// Median smoothing cannot be in place.
CvInvoke.cvSmooth(depthImage.Ptr, smoothedDepth.Ptr, SMOOTH_TYPE.CV_MEDIAN, 5, 5,
0, 0);
if (t > 1)
{
CvInvoke.cvAbsDiff(smoothedDepth.Ptr, prevSmoothedDepth.Ptr, Diff0.Ptr);
//CvInvoke.cvErode(Diff0.Ptr, Diff0.Ptr, StructuringElement.Ptr, 1);
DiffMask0.CopyTo(DiffMask1);
CvInvoke.cvThreshold(Diff0.Ptr, DiffMask0.Ptr, 2, 255, THRESH.CV_THRESH_BINARY);
if (t > 2)
{
// Makes diffMask1 the motion mask at t - 1.
CvInvoke.cvAnd(DiffMask0.Ptr, DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero);
if (bufferSize <= 1)
{
// Makes diffMask1 the motion mask at t - 0.
CvInvoke.cvXor(DiffMask0.Ptr, DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero);
}
CvInvoke.cvMorphologyEx(DiffMask1.Ptr, DiffMask1.Ptr, IntPtr.Zero, IntPtr.Zero,
CV_MORPH_OP.CV_MOP_OPEN, 1);
ComputeCumulativeDist(Diff0, diffCumulativeDist);
ComputeCumulativeDist(TrackedDepthFrame, depthCumulativeDist);
CvInvoke.cvZero(SaliencyProb);
var diffMaskData = DiffMask1.Data;
var diffData = Diff0.Data;
var depthData = TrackedDepthFrame.Data;
var probData = SaliencyProb.Data;
for (int i = 0; i < DiffMask1.Height; i++)
{
for (int j = 0; j < DiffMask1.Width; j++)
{
if (diffMaskData[i, j, 0] > 0)
{
var diffBin = diffData[i, j, 0];
var depthBin = depthData[i, j, 0];
probData[i, j, 0] = diffCumulativeDist[diffBin] * depthCumulativeDist[depthBin];
}
}
}
PrevBoundingBoxes = FindBestBoundingBox(depthFrame, skeleton);
if (PrevBoundingBoxes.LastOrDefault().Width > 0)
{
var handSkeletonPoint = SkeletonUtil.DepthToSkeleton(PrevBoundingBoxes.Last(),
TrackedDepthFrame.Data, width, height, mapper);
relPos = new Some <Vector3D>(SkeletonUtil.RelativePosToShoulder(handSkeletonPoint,
skeleton));
angle = new Some <Vector3D>(SkeletonUtil.PointDirection(handSkeletonPoint,
SkeletonUtil.GetJoint(skeleton, JointType.ElbowRight).Position));
}
}
}
List <Rectangle> colorBBs = new List <Rectangle>();
foreach (var bb in PrevBoundingBoxes)
{
var colorBox = mapper.MapDepthRectToColorRect(bb, depthFrame, width, height);
playerDetector.SmoothSkin(colorBox);
colorBBs.Add(colorBox);
}
return(new TrackingResult(relPos, angle, TrackedDepthFrame, PrevBoundingBoxes,
playerDetector.SkinImage, colorBBs));
}
void ProcessFeature()
{
IHandTracker handTracker = null;
Int16[] depthPixelData = null;
Byte[] colorPixelData = null;
Log.DebugFormat("Start processing {0}...", inputFile);
int frameCount = replayer.GetFramesCount();
for (float i = 0; i < frameCount; i += sampleRate)
{
int index = (int)Math.Round(i);
if (index >= frameCount)
{
break;
}
var skeletonFrame = replayer.GetSkeletonFrame(index);
var depthFrame = replayer.GetDepthFrame(index);
var colorFrame = replayer.GetColorFrame(index);
if (handTracker == null)
{
handTracker = (IHandTracker)Activator.CreateInstance(handTrackerType, new Object[] {
depthFrame.Width, depthFrame.Height, GetKinectParams(), bufferSize
});
}
if (featureProcessor == null)
{
featureProcessor = (IFeatureProcessor)Activator.CreateInstance(
featureProcessorType, new Object[] { sampleRate });
}
if (depthPixelData == null)
{
depthPixelData = new Int16[depthFrame.PixelDataLength];
}
if (colorPixelData == null)
{
colorPixelData = new Byte[colorFrame.PixelDataLength];
}
depthFrame.CopyPixelDataTo(depthPixelData);
colorFrame.CopyPixelDataTo(colorPixelData);
var skeleton = SkeletonUtil.FirstTrackedSkeleton(skeletonFrame.Skeletons);
var result = handTracker.Update(depthPixelData, colorPixelData, skeleton);
Option <Array> feature = featureProcessor.Compute(result);
if (feature.IsSome)
{
if (replayerType == typeof(KinectAllFramesReplay))
{
frameList.Add(depthFrame.GetFrameNumber());
}
else
{
int curIndex = (int)Math.Round(i - sampleRate * (bufferSize - 1));
frameList.Add(curIndex);
}
featureList.Add(feature.Value);
}
}
Log.DebugFormat("Finished processing {0}.", inputFile);
}
