/// <summary>
/// Extract the concatenated feature vector from a skeleton
/// </summary>
/// <param name="s">A processed Skeleton</param>
/// <returns>The concatenated feature vector</returns>
public static double[] FeatureSet(Skeleton s)
{
SkeletonFeatureFrame sff = new SkeletonFeatureFrame(s);
var features = new double[31];
int nextFreeIndex = 0;
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.ArmLeft, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.ArmRight, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.ForeArmLeft, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.ForeArmLeft, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.HandLeft, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.HandRight, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.ShoulderBladeLeft, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.ShoulderBladeRight, features, nextFreeIndex);
nextFreeIndex = AddFeat(sff, SkeletonFeatureFrame.DirectionVectors.BetweenHands, features, nextFreeIndex);
features[nextFreeIndex++] = sff.Angle(SkeletonFeatureFrame.Angles.AngleElbowLeft);
features[nextFreeIndex++] = sff.Angle(SkeletonFeatureFrame.Angles.AngleElbowRight);
features[nextFreeIndex++] = sff.Angle(SkeletonFeatureFrame.Angles.AngleShoulderLeft);
features[nextFreeIndex++] = sff.Angle(SkeletonFeatureFrame.Angles.AngleShoulderRight);
if (nextFreeIndex != features.Length) throw new Exception("Missing features");
return features;
}
public void TestHighEntropyIsRecorded()
{
// provide a sequence of high entropies followed by silence
bool eventRaised = false;
int actualHighEntropyFrames = 0;
const int expectedHighEntropyFrames = 10;
List<double[]> expectedSeq = new List<double[]>();
SequenceRecorder.Get().RecordSequence(
s => { eventRaised = true; AssertSequencesAreEqual(expectedSeq.ToArray(), s.SkeletonFeatures); },
i => { },
() => { actualHighEntropyFrames = actualHighEntropyFrames + 1; });
for (int i = 0; i < expectedHighEntropyFrames; i++)
{
prototypeSkeleton = Clone(prototypeSkeleton);
prototypeSkeleton.WristLeft.X += 100;
prototypeSkeleton.WristLeft.Y += 100;
prototypeSkeleton.WristLeft.Z += 100;
expectedSeq.Add(SkeletonDataAdapter.FeatureSet(prototypeSkeleton));
((MockKinectHandler)KinectHandler.Get()).Invoke(prototypeSkeleton);
}
for (int i = 0; i <= SequenceRecorder.LowEntropyCountNeeded + 3; i++)
{
expectedSeq.Add(SkeletonDataAdapter.FeatureSet(prototypeSkeleton));
((MockKinectHandler)KinectHandler.Get()).Invoke(prototypeSkeleton);
}
Assert.IsTrue(eventRaised);
Assert.AreEqual(expectedHighEntropyFrames, actualHighEntropyFrames);
}
public virtual PreprocessedFrame Preprocess(Skeleton skel, byte[] video, byte[] depth)
{
// process the hands, left then right:
var left = ProcessHand(skel.PalmLeft, skel.WristLeft, video, depth);
var right = ProcessHand(skel.PalmRight, skel.WristRight, video, depth);
//var features = SkeletonFeatureFrame.SkeletonDataToFeatureVector(skel);
var features = SkeletonDataAdapter.FeatureSet(skel);
return new PreprocessedFrame(features, left.Item1, right.Item1, left.Item2, right.Item2);
}
public void Setup()
{
skeleton = new Skeleton
{
ShoulderCenter = Vector3D.Zero,
ShoulderLeft = Vector3D.UnitX,
ShoulderRight = -Vector3D.UnitX
};
skeleton.ElbowLeft = skeleton.ShoulderLeft + 2 * Vector3D.UnitX;
skeleton.WristLeft = skeleton.ElbowLeft + (Vector3D.UnitY);
skeleton.PalmLeft = skeleton.WristLeft + (0.5 * Vector3D.UnitZ);
skeleton.ElbowRight = skeleton.ShoulderRight + new Vector3D(-1.5, 0, -1.5);
skeleton.WristRight = skeleton.ElbowRight + new Vector3D(0, 3.5, -3.5);
skeleton.PalmRight = skeleton.WristRight + new Vector3D(-0.5, 1, -1);
skeletonFeatureFrame = new SkeletonFeatureFrame(skeleton);
}
protected void InvokeSkeletonReady(Skeleton skel)
{
SkeletonReady(skel);
}
public SkeletonFeatureFrame(Skeleton skeleton)
{
featureVectors = FeatureVectorsOf(skeleton);
featureAngles = FeatureAnglesOf(featureVectors);
}
private static Vector3D[] FeatureVectorsOf(Skeleton sk)
{
var result = new Vector3D[Enum.GetValues(typeof(DirectionVectors)).Length];
result[(int)DirectionVectors.HandLeft] = DirectionVector(sk.WristLeft, sk.PalmLeft);
result[(int)DirectionVectors.ForeArmLeft] = DirectionVector(sk.ElbowLeft, sk.WristLeft);
result[(int)DirectionVectors.ArmLeft] = DirectionVector(sk.ShoulderLeft, sk.ElbowLeft);
result[(int)DirectionVectors.ShoulderBladeLeft] = DirectionVector(sk.ShoulderCenter, sk.ShoulderLeft);
result[(int)DirectionVectors.HandRight] = DirectionVector(sk.WristRight, sk.PalmRight);
result[(int)DirectionVectors.ForeArmRight] = DirectionVector(sk.ElbowRight, sk.WristRight);
result[(int)DirectionVectors.ArmRight] = DirectionVector(sk.ShoulderRight, sk.ElbowRight);
result[(int)DirectionVectors.ShoulderBladeRight] = DirectionVector(sk.ShoulderCenter, sk.ShoulderRight);
result[(int)DirectionVectors.BetweenHands] = DirectionVector(sk.PalmRight, sk.PalmLeft);
return result;
}
/// <summary>
/// Convert a Skeleton into a list of vectors, by filtering out only the
/// most relevant body parts (for calculating movement).
/// </summary>
/// <param name="skel">A processed skeleton</param>
/// <returns>A list containing the most relevant body parts' positions</returns>
public static List<Vector3D> ToVectorList(Skeleton skel)
{
// only consider the 4 key parts : { Elbow, Wrist } x { Left, Right }
return new List<Vector3D> { skel.ElbowLeft, skel.WristLeft, skel.ElbowRight, skel.WristRight };
}
public override PreprocessedFrame Preprocess(Skeleton skel, byte[] video, byte[] depth)
{
//var features = SkeletonFeatureFrame.SkeletonDataToFeatureVector(skel);
var features = SkeletonDataAdapter.FeatureSet(skel);
return new PreprocessedFrame(features, null, null, null, null);
}
public void Invoke(Skeleton skel)
{
InvokeSkeletonReady(skel);
}
private static Skeleton Clone(Skeleton cur)
{
return new Skeleton
{
ShoulderCenter = Clone(cur.ShoulderCenter),
ShoulderLeft = Clone(cur.ShoulderLeft),
ShoulderRight = Clone(cur.ShoulderRight),
ElbowLeft = Clone(cur.ElbowLeft),
ElbowRight = Clone(cur.ElbowRight),
WristLeft = Clone(cur.WristLeft),
WristRight = Clone(cur.WristRight),
PalmLeft = Clone(cur.PalmLeft),
PalmRight = Clone(cur.PalmRight)
};
}
private void SkeletonFrameReady(Skeleton skel)
{
if (skel == null || lastVideo == null || lastDepth == null) return;
frameHistory.Push(EntropyMonitor.ToVectorList(skel));
frames.Add(Preprocessor.Preprocess(skel, lastVideo, lastDepth));
ProcessEntropy();
}
