/// format the real time gesture data to strings
private void GenerateGestureDataString()
{
m_showStr = "";
string stateStr = "Key frame detect: ";
GestureDetector gestDet = m_manager[m_gestIdx];
for (int i = 0; i < gestDet.JointCount; i++)
{
Joint j = gestDet.m_jointIdx[i];
Vector3 vec = m_manager.m_postFeature.m_jointVec[(int)j];
m_showStr += string.Format("{0}: {1}\n", j.ToString(), vec.ToString());
}
m_showStr += '\n';
m_showStr += "Gesture name: " + m_manager[m_gestIdx].m_name + " (use left/right arrow key to switch)\n";
m_showStr += "Matching score of joints in next key posture: \n";
int g = gestDet.CurState;
//for(int i = 0; i < gestDet.PostureCount; i++) // if all postures are showed, they must be all updated in GestureDetector.Detect()
// or we only show the info for the next posture of gestDet[g]
//{
m_showStr += gestDet[g].m_name + ": ";
for (int j = 0; j < gestDet.JointCount; j++)
{
m_showStr += string.Format(" {0:F2},", gestDet[g].JointScore[j]);
}
m_showStr += "\n";
stateStr += Convert.ToByte(gestDet.IsPostDet[g]) + ",";
//}
m_showStr += string.Format("State: {0}, {1:F3}s\n", gestDet.Fsm.m_state, gestDet.Fsm.m_stateContTime);
m_showStr += stateStr + (gestDet.IsDetected ? "Bingo!" : "");
}
/// @brief Utility method to calculate the rotation of a joint
///
/// This method receives joint information and calculates the rotation of the joint in Unity
/// coordinate system.
/// @param centerOffset the new central position
/// @param joint the joint we want to calculate the rotation for
/// @param skelTrans the new transformation of the joint
/// @return the rotation of the joint in Unity coordinate system
protected Quaternion CalcRotationForJoint(Joint joint)
{
// In order to convert the skeleton's orientation to Unity orientation we will
// use the Quaternion.LookRotation method to create the relevant rotation Quaternion.
// for Quaternion.LookRotation to work it needs a "forward" vector and an "upward" vector.
// These are generally the "Z" and "Y" axes respectively in the sensor's coordinate
// system. The orientation received from the skeleton holds these values in their
// appropriate members.
// Get the forward axis from "z".
JointData jd = m_jointData[(int)joint];
Vector3 worldForward = jd.m_orient[2];
worldForward *= -1.0f; // because the Unity "forward" axis is opposite to the world's "z" axis.
if (worldForward.magnitude == 0)
{
return(Quaternion.identity); // we don't have a good point to work with.
}
// Get the upward axis from "Y".
Vector3 worldUpwards = jd.m_orient[1];
if (worldUpwards.magnitude == 0)
{
return(Quaternion.identity); // we don't have a good point to work with.
}
Quaternion jointRotation = Quaternion.LookRotation(worldForward, worldUpwards);
Quaternion newRotation = transform.rotation * jointRotation * m_jointsInitialRotations[(int)joint];
// we try to limit the speed of the change.
float t = Time.deltaTime * m_rotationDampening;
return(Quaternion.Slerp(m_jointTransforms[(int)joint].rotation, newRotation, t));
}
public void RegisterJoint(Joint j)
{
if (!m_jointsNeedUpdate.Contains(j))
{
m_jointsNeedUpdate.Add(j);
}
}
/// called by LoadFileNew and LoadFileAppend
private void ReadFileHeader(StreamReader reader,
out int frameCount, out int jointCount, out Joint[] jointIdx, out int dataPerJoint)
{
string str;
str = reader.ReadLine();
str = reader.ReadLine();
str = reader.ReadLine();
str = reader.ReadLine();
str = reader.ReadLine();
frameCount = Convert.ToInt32(str);
str = reader.ReadLine();
jointCount = Convert.ToInt32(str);
jointIdx = new Joint[jointCount];
str = reader.ReadLine();
string[] d = str.Split(',');
for (int i = 0; i < jointCount; i++)
{
jointIdx[i] = (Joint)Enum.Parse(typeof(Joint), d[i]);
}
str = reader.ReadLine(); // dataPerJoint
dataPerJoint = Convert.ToInt32(str);
}
// only the reference is copied from the recorded joint data to the joint array to compute the features!
private void CopyCurFrameJointDataRef(int frameIdx)
{
JointData[] jda = m_skRecorder.m_dataList[frameIdx];
for (int i = 0; i < m_skRecorder.JointCount; i++)
{
Joint j = m_skRecorder.m_jointIdx[i];
m_postFeature.m_jointData[(int)j] = jda[i];
}
}
// only the reference is copied from the recorded joint data to the joint array to control the avatar!
private void CopyCurFrameJointDataRef(int frameIdx)
{
if (frameIdx > m_skRecorder.FrameCount - 1)
{
return;
}
JointData[] jda = m_skRecorder.m_dataList[frameIdx];
for (int i = 0; i < m_skRecorder.JointCount; i++)
{
Joint j = m_skRecorder.m_jointIdx[i];
m_avJointData[(int)j] = jda[i];
}
}
/// @brief a utility method to update joint position
///
/// This utility method receives a joint and unscaled position (x,y,z) and moves the joint there.
/// it makes sure the joint has been attached and that scale is applied.
/// @param joint The joint to update (the method makes sure it is legal)
/// @param xPos The unscaled position along the x axis (scale will be applied)
/// @param yPos The unscaled position along the y axis (scale will be applied)
/// @param zPos The unscaled position along the z axis (scale will be applied)
protected void UpdateJointPosition(Joint joint, float xPos, float yPos, float zPos)
{
if (((int)joint) >= m_jointTransforms.Length || m_jointTransforms[(int)joint] == null)
{
return; // an illegal joint
}
Vector3 tmpPos = Vector3.zero;
tmpPos.x = xPos;
tmpPos.y = yPos;
tmpPos.z = zPos;
tmpPos *= m_scale;
m_jointTransforms[(int)joint].localPosition = tmpPos;
}
private void UpdateKinectJointData(OpenNI.SkeletonJoint joint, int player, ref JointData jointData)
{
OpenNI.SkeletonJointTransformation data;
if (!playerManager.GetPlayer(player).GetSkeletonJoint(joint, out data))
{
return;
}
jointData.position = coordinateSystem.ConvertLocation(coordinateSystem.ConvertRawKinectLocation(data.Position.Position), RUISDevice.Kinect_1);
jointData.positionConfidence = data.Position.Confidence;
jointData.rotation = coordinateSystem.ConvertRotation(coordinateSystem.ConvertRawKinectRotation(data.Orientation), RUISDevice.Kinect_1);
jointData.rotationConfidence = data.Orientation.Confidence;
}
/// assign m_jointStartIdx and m_jointEndIdx
private static void InitJointVecIdx()
{
Joint head = Joint.Head,
neck = Joint.Neck, tors = Joint.Torso,
lShd = Joint.LeftShoulder, lElb = Joint.LeftElbow, lHnd = Joint.LeftHand,
lHip = Joint.LeftHip, lKne = Joint.LeftKnee, lFot = Joint.LeftFoot,
rShd = Joint.RightShoulder, rElb = Joint.RightElbow, rHnd = Joint.RightHand,
rHip = Joint.RightHip, rKne = Joint.RightKnee, rFot = Joint.RightFoot;
Joint[] startJointIdx = { head, neck, lShd, tors, lHip, neck, rShd, tors, rHip };
Joint[] currentJointIdx = { neck, lShd, lElb, lHip, lKne, rShd, rElb, rHip, rKne };
Joint[] endJointIdx = { tors, lElb, lHnd, lKne, lFot, rElb, rHnd, rKne, rFot };
for (int i = 0; i < currentJointIdx.Length; i++)
{
m_jointStartIdx[(int)currentJointIdx[i]] = startJointIdx[i];
m_jointEndIdx[(int)currentJointIdx[i]] = endJointIdx[i];
}
}
/// generate strings for UIInfoObserver
public void GetObserverString(out string strData, out string strFrame)
{
strData = "Joint info\n";
if (m_skRecorder.FrameCount > 0)
{
JointData[] jda = m_skRecorder.m_dataList[m_curFrame];
for (int i = 0; i < m_skRecorder.JointCount; i++)
{
Joint j = m_skRecorder.m_jointIdx[i];
Vector3 vec = jda[i].m_pos;
strData += string.Format("{0}: {1}\n", j.ToString(), vec.ToString());
}
strData += string.Format("\nFrame {0}/{1}, time {2} s\n", m_curFrame + 1, m_skRecorder.FrameCount, m_skRecorder.m_timeList[m_curFrame]);
if (m_skRecorder.m_tagList[m_curFrame] != 0)
{
strData += string.Format("key frame tag {0}", m_skRecorder.m_tagList[m_curFrame]);
}
}
strFrame = m_strFrame;
}
void DrawLineBetweenJoints(OpenNI.SkeletonJoint first, OpenNI.SkeletonJoint second)
{
NISelectedPlayer player = playerSelection.GetPlayer(0);
OpenNI.SkeletonJointPosition firstJointPosition;
player.GetSkeletonJointPosition(first, out firstJointPosition);
OpenNI.SkeletonJointPosition secondJointPosition;
player.GetSkeletonJointPosition(second, out secondJointPosition);
if (firstJointPosition.Confidence <= 0.5 || secondJointPosition.Confidence <= 0.5)
{
return;
}
OpenNI.Point3D firstJointScreenPosition = depthGenerator.ConvertRealWorldToProjective(firstJointPosition.Position);
OpenNI.Point3D secondJointScreenPosition = depthGenerator.ConvertRealWorldToProjective(secondJointPosition.Position);
DrawLine.DrawSimpleLine(ref mapPixels,
(int)(width - firstJointScreenPosition.X / factor), (int)(height - firstJointScreenPosition.Y / factor),
(int)(width - secondJointScreenPosition.X / factor), (int)(height - secondJointScreenPosition.Y / factor),
width, height,
Color.white);
}
/// @brief updates a single joint
///
/// This method updates a single joint. The decision of what to update (orientation, position)
/// depends on m_updateOrientation and m_updateJointPositions. Only joints with high confidence
/// are updated. @note it is possible to update only position or only orientation even though both
/// are expected if the confidence of one is low.
/// @param centerOffset the new central position
/// @param joint the joint we want to update
/// @param skelTrans the new transformation of the joint
protected void UpdateJoint(Joint joint)
{
// make sure something is hooked up to this joint
if ((int)joint >= m_jointTransforms.Length || !m_jointTransforms[(int)joint])
{
return;
}
JointData jd = m_jointData[(int)joint];
Quaternion rot = CalcRotationForJoint(joint);
Vector3 pos0 = (jd.m_pos - m_centerOffset) * m_scale;
// if we have debug lines to draw we need to collect the data.
if (m_isDebugLineActive & m_linesDebugger != null)
{
Vector3 pos = pos0 * 1.2f + // magnify the scale of the debugger by 1.2 times
transform.position +
new Vector3(1, 0, 0); // add a offset, from the avatar
float posConf = jd.m_posConf;
float rotConf = jd.m_orientConf;
m_linesDebugger.UpdateJointInfoForJoint(joint, pos, posConf, rot, rotConf);
}
// modify orientation (if needed and confidence is high enough)
// the confidence of the hands are always 0, maybe also ankles
if (m_updateOrientation && jd.m_orientConf >= 0.5 && joint != Joint.LeftHand && joint != Joint.RightHand)
{
m_jointTransforms[(int)joint].rotation = rot;
}
// modify position (if needed, and confidence is high enough)
if (m_updateJointPositions && jd.m_posConf >= 0.5)
{
m_jointTransforms[(int)joint].localPosition = pos0; // CalcJointPosition(jd);
}
}
