void buildBody(GameObject parent)
{
int count = 0;
foreach (string entry in jointMap)
{
// do something with entry.Value or entry.Key
// assuming it has a line renderer
GameObject jointObj = GameObject.CreatePrimitive(PrimitiveType.Sphere);
jointObj.GetComponent <Renderer>().material.shader = Shader.Find("Diffuse");
// no line rendered for the head
if (!entry.Equals("Head"))
{
LineRenderer lr = jointObj.AddComponent <LineRenderer>();
lr.SetVertexCount(2);
lr.material = BoneMaterial;
lr.SetWidth(LineThickness, LineThickness);
}
jointObj.transform.localScale = parent.transform.localScale / 2;
jointObj.name = entry;
jointObj.transform.parent = parent.transform;
// now for the joint struct
if (joints[count] == null)
{
joints[count] = new KinectJoint();
joints[count].name = jointObj.name;
count++;
}
}
}
public void SetBodyData(Body sourceBody, Quaternion kinectRot, float kinectHeight)
{
trackingId = sourceBody.TrackingId;
isTracked = sourceBody.IsTracked;
if (!isTracked)
{
return;
}
if (jointArray == null || jointArray.Length != sourceBody.Joints.Count)
{
jointArray = sourceBody.Joints.Select(pair =>
{
var joint = new KinectJoint();
joint.type = pair.Key;
return(joint);
}).ToArray();
joints = jointArray.ToDictionary(j => j.type, j => j);
}
foreach (var key in sourceBody.Joints.Keys)
{
var sourceJoint = sourceBody.Joints[key];
var sourceOrientation = sourceBody.JointOrientations[key];
joints[key].SetJointData(sourceJoint, sourceOrientation, kinectRot, kinectHeight);
}
handLeftState = sourceBody.HandLeftState;
handRightState = sourceBody.HandRightState;
}
public OrientationInfo GetJointInfo(KinectJoint joint, int?playerId = null)
{
if (this.thisKinectGesturePlugin == null)
{
return(null);
}
if (this.thisKinectGesturePlugin.gestureProcessor == null)
{
return(null);
}
if (this.thisKinectGesturePlugin.gestureProcessor.Skeletons == null)
{
return(null);
}
Microsoft.Kinect.Skeleton skeleton = null;
if (playerId == null)
{
foreach (Microsoft.Kinect.Skeleton s in this.thisKinectGesturePlugin.gestureProcessor.Skeletons)
{
if (s.TrackingState != Microsoft.Kinect.SkeletonTrackingState.NotTracked)
{
skeleton = s; break;
}
}
}
else
{
foreach (Microsoft.Kinect.Skeleton s in this.thisKinectGesturePlugin.gestureProcessor.Skeletons)
{
if (s.TrackingId == playerId.Value)
{
skeleton = s; break;
}
}
}
if (skeleton != null)
{
Microsoft.Kinect.SkeletonPoint jointPos = this.thisKinectGesturePlugin.gestureProcessor.NormalizeJoint(skeleton.Joints[(Microsoft.Kinect.JointType)joint]);
return(new OrientationInfo()
{
playerId = skeleton.TrackingId,
tracked = (skeleton.Joints[(Microsoft.Kinect.JointType)joint].TrackingState != Microsoft.Kinect.JointTrackingState.NotTracked),
X = jointPos.X,
Y = jointPos.Y,
Z = jointPos.Z,
});
}
else
{
return(null);
}
}
/// <summary>
/// Maps the information received for a Body from the Kinect to a
/// KinectBody we can serialize and send over the wire.
/// </summary>
/// <param name="body">Body to map</param>
/// <param name="sensorId">Sensor ID, or any other value being used as the device connection Id</param>
/// <returns>Mapped KinectBody containing the body information,
/// an identifier, and othe processed data</returns>
private static KinectBody MapBody(Body body, string sensorId)
{
var d = new KinectBody(sensorId, body.TrackingId);
// All six bodies are fully tracked. Wee!
var jointCount = Enum.GetNames(typeof(KinectJointType)).Length;
d.Joints = new KinectJoint[jointCount];
for (var i = 0; i < jointCount; i++)
{
var nativeJoint = body.Joints[(JointType)i];
var orientation = body.JointOrientations[(JointType)i].Orientation;
var joint = new KinectJoint
{
TrackingState = ((KinectTrackingState)(int)nativeJoint.TrackingState),
Position = new KinectVector3
{
X = nativeJoint.Position.X,
Y = nativeJoint.Position.Y,
Z = nativeJoint.Position.Z
},
JointType = ((KinectJointType)(int)nativeJoint.JointType),
Orientation = new KinectVector4
{
W = orientation.W,
X = orientation.X,
Y = orientation.Y,
Z = orientation.Z
}
};
d.Joints[i] = joint;
}
d.Lean = new KinectPoint {
X = body.Lean.X, Y = body.Lean.Y
};
d.LeanTrackingState = (KinectTrackingState)(int)body.LeanTrackingState;
// Record hand states
d.HandLeftState = (KinectHandState)(int)body.HandLeftState;
d.HandRightState = (KinectHandState)(int)body.HandRightState;
// Record hand confidence. Initially we'll just convert the enum to an int,
// but we could do some exponential smoothing between their {0,1} values.
d.HandLeftConfidence = (int)body.HandLeftConfidence;
d.HandRightConfidence = (int)body.HandRightConfidence;
return(d);
}
/// <summary>
/// The Kinect Object constructor
/// </summary>
/// <param name="firstJoint">The first joint</param>
/// <param name="secondJoint">The second joint</param>
/// <param name="image">The image to be drawn</param>
/// <param name="offsetX">The offset X</param>
/// <param name="offsetY">The offset Y</param>
public KinectObject(KinectJoint firstJoint, KinectJoint secondJoint, ImageSource image, int offsetX = 0, int offsetY = 0)
{
this.FirstJoint = (int)firstJoint;
this.SecondJoint = (int)secondJoint;
this.Image = image;
if (offsetX != 0)
{
OffsetX = offsetX;
}
if (offsetY != 0)
{
OffsetY = offsetY;
}
}
public override void Redraw(params object[] args)
{
if (active)
{
object initialAngle = args[0];
double currentAngle = (double)args[1];
KinectJoint activeJoint = (KinectJoint)args[2];
angleText.text = string.Format("{0} °", currentAngle.ToString("0"));
jointName.text = activeJoint.Translation;
referenceCircle.Redraw(initialAngle);
currentCircle.Redraw(currentAngle);
}
}
public void DrawDebug(Kinect.Body body, KinectJoint patientJoint)
{
var debugText = debugTexts[patientJoint.Name];
var joint = body.Joints[patientJoint.Type];
var angle = Calculations.GetAngle(
joint, body.Joints[patientJoint.Parent.Type], body.Joints[patientJoint.Children.First().Value.Type]
);
var position = Calculations.GetVector3FromJoint(joint);
position.Set(position.x, position.y, position.z - 1);
debugText.transform.position = position;
debugText.text = string.Format("Angle: {0}°\nX: {1}\nY: {2}\nZ: {3}", angle.ToString("0"), joint.Position.X.ToString("0.000"), joint.Position.Y.ToString("0.000"), joint.Position.Z.ToString("0.000"));
}
public float GetRelationshipInfo(KinectJoint actor, KinectJointRelationship relation, string relative)
{
if (this.thisKinectGesturePlugin == null)
{
return(float.NaN);
}
if (this.thisKinectGesturePlugin.gestureProcessor == null)
{
return(float.NaN);
}
if (this.thisKinectGesturePlugin.gestureProcessor.Skeletons == null)
{
return(float.NaN);
}
Microsoft.Kinect.Skeleton skeleton = null;
foreach (Microsoft.Kinect.Skeleton s in this.thisKinectGesturePlugin.gestureProcessor.Skeletons)
{
if (s.TrackingState != Microsoft.Kinect.SkeletonTrackingState.NotTracked)
{
skeleton = s; break;
}
}
if (skeleton != null)
{
if (this.thisKinectGesturePlugin.gestureProcessor.Relationships.ContainsKey(skeleton.TrackingId))
{
if (this.thisKinectGesturePlugin.gestureProcessor.Relationships[skeleton.TrackingId].ContainsKey((Microsoft.Kinect.JointType)actor))
{
if (this.thisKinectGesturePlugin.gestureProcessor.Relationships[skeleton.TrackingId][(Microsoft.Kinect.JointType)actor].ContainsKey(relative))
{
if (this.thisKinectGesturePlugin.gestureProcessor.Relationships[skeleton.TrackingId][(Microsoft.Kinect.JointType)actor][relative].ContainsKey((GestureParser.Relationship)relation))
{
return(this.thisKinectGesturePlugin.gestureProcessor.Relationships[skeleton.TrackingId][(Microsoft.Kinect.JointType)actor][relative][(GestureParser.Relationship)relation]);
}
return(float.NaN);
}
return(float.NaN);
}
return(float.NaN);
}
return(float.NaN);
}
return(float.NaN);
}
/// <summary>
/// Gets the "in scene" coordinated of the specified joint. The coordinated are for the 640x480
/// depth image
/// </summary>
/// <param name="userId">The ID of the specific user</param>
/// <param name="joint">Name of the joint from the KinectJoint enum</param>
/// <returns>Coordinates of the joint</returns>
public PointF getJointScene(UInt64 userId, KinectJoint joint)
{
double x, y, z;
bool result;
if (!Native.PoseRecognizer_getUserJointScene(instance, userId, (int)joint, out x, out y, out z, out result))
{
throw new NullReferenceException("Error while trying to read joint coordinates!");
}
if (result)
{
return(new PointF((float)x, (float)y));
}
else
{
return(new PointF(0, 0));
}
}
/// <summary>
/// Gets the real world coordinates of the specified joint
/// </summary>
/// <param name="userId">The ID of the specific user</param>
/// <param name="joint">Name of the joint</param>
/// <returns>Coordinates of a joint as 3D-point</returns>
public cvPoint3f getJointRealworld(UInt64 userId, KinectJoint joint)
{
double x, y, z;
bool result;
if (!Native.PoseRecognizer_getUserJointRealworld(instance, userId, (int)joint, out x, out y, out z, out result))
{
throw new NullReferenceException("Error while trying to read joint coordinates!");
}
if (result)
{
return(new cvPoint3f(x, y, z));
}
else
{
return(new cvPoint3f(0, 0, 0));
}
}
private void Sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
using (var frame = e.OpenSkeletonFrame())
{
if (frame != null)
{
frame.CopySkeletonDataTo(skeleton);
kinectJoints = new List <KinectJoint>();
foreach (var body in skeleton)
{
if (body.TrackingState == SkeletonTrackingState.Tracked && !bodyDeceted)
{
form.BodyDetected();
bodyDeceted = true;
}
if (body.TrackingState == SkeletonTrackingState.Tracked && saveCoordinates)
{
foreach (Joint joint in body.Joints)
{
SkeletonPoint skeletonPoint = joint.Position;
KinectJoint kinectJoint = new KinectJoint();
if (joint.JointType.ToString().Equals(HAND_LEFT) || joint.JointType.ToString().Equals(HAND_RIGHT))
{
kinectJoint.Type = joint.JointType.ToString();
kinectJoint.Moment = moment;
kinectJoint.X = skeletonPoint.X;
kinectJoint.Y = skeletonPoint.Y;
kinectJoint.Z = skeletonPoint.Z;
kinectJoints.Add(kinectJoint);
jointCount++;
}
}
jointsInMoment.Add(moment.ToString(), kinectJoints);
moment++;
}
}
}
}
}
// Exposes the library's ability to add a gesture step's failure condition
public int AddGestureStepFailureRelationship(KinectJoint actor, KinectJointRelationship relationship, string relative, int deviation, int stepNumber = -1, string gestureName = null)
{
if (gestureName != null)
{
lastGesture = GetGesture(gestureName);
}
if (stepNumber != -1)
{
lastStep = stepNumber;
}
GestureParser.JointRelationship jr = new GestureParser.JointRelationship()
{
actor = (Microsoft.Kinect.JointType)actor,
relation = (GestureParser.Relationship)relationship,
relative = relative,
deviation = deviation
};
lastGesture.steps.ElementAt(lastStep).FailureConditions.Add(jr);
lastFailureCondition = (lastGesture.steps.ElementAt(lastStep).FailureConditions.Count - 1);
return(lastFailureCondition);
}
void buildBodyParticles(GameObject parent)
{
int count = 0;
foreach (string entry in jointMap)
{
// do something with entry.Value or entry.Key
// assuming it has a line renderer
GameObject jointObj = new GameObject();
GameObject particles = (GameObject)Instantiate(ParticleSystemPrefab, jointObj.transform.position, Quaternion.identity);
particles.GetComponent <ParticleSystem>().collision.SetPlane(1, CollisionPlane.transform);
particles.transform.parent = jointObj.transform;
jointObj.transform.localScale = parent.transform.localScale / 2;
jointObj.name = entry;
jointObj.transform.parent = parent.transform;
// now for the joint struct
if (joints[count] == null)
{
joints[count] = new KinectJoint();
joints[count].name = jointObj.name;
count++;
}
}
// Extra particles
foreach (KeyValuePair <string, string> entry in boneMap)
{
GameObject jointObj = new GameObject();
jointObj.name = entry.Value + "_" + entry.Key;
jointObj.transform.parent = parent.transform;
GameObject particles = (GameObject)Instantiate(ParticleSystemPrefab, Vector3.zero, Quaternion.identity);
particles.GetComponent <ParticleSystem>().collision.SetPlane(1, CollisionPlane.transform);
particles.transform.parent = jointObj.transform;
}
}
void buildBodyBox(GameObject parent)
{
int count = 0;
// Create the joints
foreach (string entry in jointMap)
{
// do something with entry.Value or entry.Key
// assuming it has a line renderer
GameObject jointObj = new GameObject();
jointObj.transform.localScale = parent.transform.localScale / 2;
jointObj.name = entry;
jointObj.transform.parent = parent.transform;
// now for the joint struct
if (joints[count] == null)
{
joints[count] = new KinectJoint();
joints[count].name = jointObj.name;
count++;
}
}
// Create the boxes
foreach (KeyValuePair <string, string> entry in boneMap)
{
GameObject jointParent = this.transform.FindChild(entry.Value).gameObject;
GameObject joint = this.transform.FindChild(entry.Key).gameObject;
GameObject box = GameObject.CreatePrimitive(PrimitiveType.Cube);
box.GetComponent <Renderer>().material.shader = Shader.Find("Diffuse");
box.name = entry.Value + "_" + entry.Key;
box.transform.parent = parent.transform;
}
}
// Exposes the library's ability to set a gesture step's failure relationship
public void SetGestureStepFailureRelationship(string gestureName, int stepNumber, int conditionNumber, KinectJoint actor, KinectJointRelationship relationship, string relative, int deviation = 0)
{
lastGesture = GetGesture(gestureName);
lastStep = stepNumber;
lastGesture.steps.ElementAt(stepNumber).FailureConditions.ElementAt(conditionNumber).actor = (Microsoft.Kinect.JointType)actor;
lastGesture.steps.ElementAt(stepNumber).FailureConditions.ElementAt(conditionNumber).relation = (GestureParser.Relationship)relationship;
lastGesture.steps.ElementAt(stepNumber).FailureConditions.ElementAt(conditionNumber).relative = relative;
lastGesture.steps.ElementAt(stepNumber).FailureConditions.ElementAt(conditionNumber).deviation = deviation;
}
public void DrawCircle(double initialAngle, double currentAngle, KinectJoint joint)
{
showCircle.Active = true;
showCircle.Redraw(initialAngle, currentAngle, joint);
}
public static KinectJoint Desserialize(byte[] data)
{
KinectJoint result = new KinectJoint();
using (MemoryStream m = new MemoryStream(data))
{
using (BinaryReader reader = new BinaryReader(m))
{
result.JointType = reader.ReadString();
result.IsTracking = reader.ReadBoolean();
result.PositionX = reader.ReadSingle();
result.PositionY = reader.ReadSingle();
result.PositionZ = reader.ReadSingle();
result.RotationX = reader.ReadSingle();
result.RotationY = reader.ReadSingle();
result.RotationZ = reader.ReadSingle();
result.RotationW = reader.ReadSingle();
}
}
return result;
}
/// <summary>
/// The constructor for the Kinect 3D Object
/// </summary>
/// <param name="firstJoint">The joint where the Model should be placed on</param>
/// <param name="model">The 3D Model</param>
public Kinect3DObject(KinectJoint firstJoint, Model3D model)
{
FirstJoint = (int)firstJoint;
Model = model;
}
public int AddGestureStepFailureRelationship(KinectJoint actor, KinectJointRelationship relationship, string relative, int deviation, int stepNumber = -1, string gestureName = null)
{
return(this.thisKinectGesturePlugin.AddGestureStepFailureRelationship(actor, relationship, relative, deviation, stepNumber, gestureName));
}
public void SetGestureStepFailureRelationship(string gestureName, int stepNumber, int conditionNumber, KinectJoint actor, KinectJointRelationship relationship, string relative, int deviation = 0)
{
this.thisKinectGesturePlugin.SetGestureStepFailureRelationship(gestureName, stepNumber, conditionNumber, actor, relationship, relative, deviation);
}
