private Quaternion RotateBone(OverlayObject oo, KinectManager kinectManager, FacetrackingManager facetrackingManager, long key)
{
int iJointIndex = (int)oo.TrackedJointType;
Quaternion qRotObject = Quaternion.identity;
if (iJointIndex == 3) // if it's the head
{
if (facetrackingManager) // && facetrackingManager.IsFaceTrackingInitialized())
{
qRotObject *= facetrackingManager.GetHeadRotation(key, bMirroredMovement: !RearProjection);
if (oo.BoneGameObject.FindChild("Jaw") != null)
{
Transform Jaw;
if (facetrackingManager.bGotAU && oo.BoneGameObject.FindChild("Jaw") != null)
{
Jaw = oo.BoneGameObject.FindChild("Jaw");
// print("Jaw angle : " + facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER);
Jaw.localRotation = Quaternion.AngleAxis(facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER, Vector3.right);
}
}
}
}
else // all joints not the head
{
qRotObject = kinectManager.GetJointOrientation(key, iJointIndex, flip: RearProjection);
Vector3 rotAngles = qRotObject.eulerAngles - oo.InitialRotation.eulerAngles;
qRotObject = Quaternion.Euler(rotAngles);
}
return(Quaternion.Slerp(oo.BoneGameObject.transform.rotation, qRotObject, smoothFactor * Time.deltaTime));
}
private Vector3 TranslateBone(OverlayObject oo, long key, KinectManager kinectManager)
{
int iJointIndex = (int)oo.TrackedJointType;
Vector3 jp = PlaceJoint(iJointIndex, key, kinectManager);
Vector3 vPosObject = Vector3.zero;
float xNorm = Mathf.Max(-10, Mathf.Min(jp.x, 10)); // addded the cap to avoid the out of camera frustrum error that resulted from infinty in these values
float yNorm = Mathf.Max(-10, Mathf.Min(jp.y, 10));
if (xNorm < -9.9f || xNorm > 9.9f)
{
return(Vector3.zero);
}
float posJointZ = jp.z;
// normalize to camera space
Vector3 foo;
if (RearProjection)
{
foo = new Vector3(1 - xNorm, yNorm, (posJointZ / ArenaDepth) * ArenaZ - Camera.main.transform.position.z);
}
else
{
foo = new Vector3(xNorm, yNorm, (posJointZ / ArenaDepth) * ArenaZ - Camera.main.transform.position.z);
}
vPosObject = Camera.main.ViewportToWorldPoint(foo);
// print(foo);
return(Vector3.Lerp(oo.BoneGameObject.position, vPosObject, smoothFactor * Time.deltaTime));
}
public Vector3 ScaleBone(KinectManager kinectManager, long key, OverlayObject oo, int iJointIndex)
{
float boneScale;
if (iJointIndex == 3)////special scale for head
{
boneScale = ScaleHead(kinectManager, key, oo, iJointIndex);
}
else if (iJointIndex >= 0 && iJointIndex <= 2 || iJointIndex == 20) /// special scale for torso & neck
{
float torsoScale = ScaleTorso(kinectManager, key, iJointIndex);
float torsoScaleY = LimbBoneScale(oo);
torsoScale *= torsoScaleY;
return(new Vector3(torsoScale, torsoScaleY, torsoScale));
}
else if (iJointIndex == (int)KinectInterop.JointType.FootLeft || iJointIndex == (int)KinectInterop.JointType.FootRight)
{
boneScale = FootBoneScale(oo);
}
else // the rest should be limbs
{
boneScale = LimbBoneScale(oo);
}
Vector3 scaleBone = new Vector3(boneScale, boneScale, boneScale);
return(scaleBone);
}
private float LimbBoneScale(OverlayObject oo)
{
Vector3 endJointPos = oo.ReferenceGameObjectTransform.position;
float boneLength = BoneLength(oo, endJointPos);
float boneScale;
boneScale = Mathf.Max(Mathf.Min(boneLength, oo.InitialBoneLength * MAXBONESCALE), oo.InitialBoneLength * MINBONESCALE) / oo.InitialBoneLength;
return(boneScale);
}
private float ScaleHead(KinectManager kinectManager, long key, OverlayObject oo, int iJointIndex)
{
float boneScale = AdultTorsoQuotient(kinectManager, key); // how much of this torso height is adult height 1 = adult 0 = child
boneScale *= -0.16666f;
boneScale += 0.5f;
float boneLength = BoneLength(oo, oo.ReferenceGameObjectTransform.position);
boneScale *= boneLength;
boneScale *= HEADSIZE;
return(boneScale);
}
public OverlayObject findParent(string name, List <OverlayObject> listofBones)
{
OverlayObject boneOO = new OverlayObject();
foreach (OverlayObject oo in listofBones)
{
if (oo.BoneGameObject.gameObject.name == name)
{//print (name);
boneOO = oo;
}
}
return(boneOO);
}
void Start()
{
CanvasEvents.OnChooserPicture += pictureButtonEnter;
if (artStyle == ArtStyle.Anatomy)
{
if (RearProjection)
{
SkeletonFBX = SkeletonFBXRearProjection;
}
else
{
SkeletonFBX = SkeletonFBXMirrored;
}
}
else if (artStyle == ArtStyle.CalacaLoca)
{
SkeletonFBX = LaCalacaLoca;
}
else if (artStyle == ArtStyle.XRay)
{
SkeletonFBX = XRaySkeleton;
}
handTipRight = SkeletonFBX.FindChild("RightHandTip");
handRight = SkeletonFBX.FindChild("RightHand");
elbowRight = SkeletonFBX.FindChild("RightElbow");
shoulderRight = SkeletonFBX.FindChild("RightShoulder");
head = SkeletonFBX.FindChild("Head");
neck = SkeletonFBX.FindChild("Neck");
handTipLeft = SkeletonFBX.FindChild("LeftHandTip");
handLeft = SkeletonFBX.FindChild("LeftHand");
elbowLeft = SkeletonFBX.FindChild("LeftElbow");
shoulderLeft = SkeletonFBX.FindChild("LeftShoulder");
spineShoulder = SkeletonFBX.FindChild("SpineShoulder");
spineMid = SkeletonFBX.FindChild("SpineMid");
spineBase = SkeletonFBX.FindChild("SpineBase");
hipRight = SkeletonFBX.FindChild("RightHip");
kneeRight = SkeletonFBX.FindChild("RightKnee");
footRight = SkeletonFBX.FindChild("RightFoot");
hipLeft = SkeletonFBX.FindChild("LeftHip");
kneeLeft = SkeletonFBX.FindChild("LeftKnee");
footLeft = SkeletonFBX.FindChild("LeftFoot");
OverlayObject handTipRightObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.HandTipRight, BoneGameObject = handTipRight };
OverlayObject handRightObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.WristRight, BoneGameObject = handRight, InitialRotation = handRight.transform.rotation, ReferenceGameObjectName = handTipRight };
OverlayObject elbowRightObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.ElbowRight, BoneGameObject = elbowRight, InitialRotation = elbowRight.transform.rotation, ReferenceGameObjectName = handRight };
OverlayObject shoulderRightObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.ShoulderRight, BoneGameObject = shoulderRight, InitialRotation = shoulderRight.transform.rotation, ReferenceGameObjectName = elbowRight };
OverlayObject hipRightObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.HipRight, BoneGameObject = hipRight, ReferenceGameObjectName = kneeRight };
OverlayObject kneeRightObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.KneeRight, BoneGameObject = kneeRight, ReferenceGameObjectName = footRight };
OverlayObject footRightObject; // = new OverlayObject (){TrackedJointType = KinectInterop.JointType.FootRight, BoneGameObject = footRight, ReferenceGameObjectName = kneeRight};
OverlayObject hipLeftObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.HipLeft, BoneGameObject = hipLeft, ReferenceGameObjectName = kneeLeft };
OverlayObject kneeLeftObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.KneeLeft, BoneGameObject = kneeLeft, ReferenceGameObjectName = footLeft };
OverlayObject footLeftObject; // = new OverlayObject () { TrackedJointType = KinectInterop.JointType.FootLeft, BoneGameObject = footLeft, ReferenceGameObjectName = kneeLeft};
OverlayObject handTipLeftObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.HandTipLeft, BoneGameObject = handTipLeft };
OverlayObject handLeftObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.WristLeft, BoneGameObject = handLeft, InitialRotation = handLeft.transform.rotation, ReferenceGameObjectName = handTipLeft };
OverlayObject elbowLeftObject; // = new OverlayObject() { TrackedJointType = KinectInterop.JointType.ElbowLeft, BoneGameObject = elbowLeft, InitialRotation = elbowLeft.transform.rotation, ReferenceGameObjectName = handLeft };
OverlayObject shoulderLeftObject;
if (RearProjection)
{
handTipRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HandTipRight, BoneGameObject = handTipRight
};
handRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.WristRight, BoneGameObject = handRight, InitialRotation = handRight.transform.rotation, ReferenceGameObjectTransform = handTipRight
};
elbowRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ElbowRight, BoneGameObject = elbowRight, InitialRotation = elbowRight.transform.rotation, ReferenceGameObjectTransform = handRight
};
shoulderRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ShoulderRight, BoneGameObject = shoulderRight, InitialRotation = shoulderRight.transform.rotation, ReferenceGameObjectTransform = elbowRight
};
hipRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HipRight, BoneGameObject = hipRight, ReferenceGameObjectTransform = kneeRight
};
kneeRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.KneeRight, BoneGameObject = kneeRight, ReferenceGameObjectTransform = footRight
};
footRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.FootRight, BoneGameObject = footRight, ReferenceGameObjectTransform = kneeRight
};
hipLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HipLeft, BoneGameObject = hipLeft, ReferenceGameObjectTransform = kneeLeft
};
kneeLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.KneeLeft, BoneGameObject = kneeLeft, ReferenceGameObjectTransform = footLeft
};
footLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.FootLeft, BoneGameObject = footLeft, ReferenceGameObjectTransform = kneeLeft
};
handTipLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HandTipLeft, BoneGameObject = handTipLeft
};
handLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.WristLeft, BoneGameObject = handLeft, InitialRotation = handLeft.transform.rotation, ReferenceGameObjectTransform = handTipLeft
};
elbowLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ElbowLeft, BoneGameObject = elbowLeft, InitialRotation = elbowLeft.transform.rotation, ReferenceGameObjectTransform = handLeft
};
shoulderLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ShoulderLeft, BoneGameObject = shoulderLeft, InitialRotation = shoulderLeft.transform.rotation, ReferenceGameObjectTransform = elbowLeft
};
}
else // non-rear projection is mirrored, so reverse the left and right game objects' respective joint assignments.
{
handTipRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HandTipRight, BoneGameObject = handTipLeft
};
handRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.WristRight, BoneGameObject = handLeft, InitialRotation = handLeft.transform.rotation, ReferenceGameObjectTransform = handTipLeft
};
elbowRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ElbowRight, BoneGameObject = elbowLeft, InitialRotation = elbowLeft.transform.rotation, ReferenceGameObjectTransform = handLeft
};
shoulderRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ShoulderRight, BoneGameObject = shoulderLeft, InitialRotation = shoulderLeft.transform.rotation, ReferenceGameObjectTransform = elbowLeft
};
hipRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HipRight, BoneGameObject = hipLeft, ReferenceGameObjectTransform = kneeLeft
};
kneeRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.KneeRight, BoneGameObject = kneeLeft, ReferenceGameObjectTransform = footLeft
};
footRightObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.FootRight, BoneGameObject = footLeft, ReferenceGameObjectTransform = kneeLeft
};
hipLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HipLeft, BoneGameObject = hipRight, ReferenceGameObjectTransform = kneeRight
};
kneeLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.KneeLeft, BoneGameObject = kneeRight, ReferenceGameObjectTransform = footRight
};
footLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.FootLeft, BoneGameObject = footRight, ReferenceGameObjectTransform = kneeRight
};
handTipLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.HandTipLeft, BoneGameObject = handTipRight
};
handLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.WristLeft, BoneGameObject = handRight, InitialRotation = handRight.transform.rotation, ReferenceGameObjectTransform = handTipRight
};
elbowLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ElbowLeft, BoneGameObject = elbowRight, InitialRotation = elbowRight.transform.rotation, ReferenceGameObjectTransform = handRight
};
shoulderLeftObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.ShoulderLeft, BoneGameObject = shoulderRight, InitialRotation = shoulderRight.transform.rotation, ReferenceGameObjectTransform = elbowRight
};
}
OverlayObject headObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.Head, BoneGameObject = head, InitialRotation = head.transform.rotation, ReferenceGameObjectTransform = spineBase
};
OverlayObject neckObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.Neck, BoneGameObject = neck, InitialRotation = neck.transform.rotation, ReferenceGameObjectTransform = head
};
OverlayObject spineShoulderObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.SpineShoulder, BoneGameObject = spineShoulder, ReferenceGameObjectTransform = neck
};
OverlayObject spineMidObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.SpineMid, BoneGameObject = spineMid, ReferenceGameObjectTransform = spineShoulder
};
OverlayObject spineBaseObject = new OverlayObject()
{
TrackedJointType = KinectInterop.JointType.SpineBase, BoneGameObject = spineBase, ReferenceGameObjectTransform = spineMid
};
OverlayObjectsExample = new List <OverlayObject>()
{
handTipRightObject, handRightObject, elbowRightObject, shoulderRightObject, headObject, neckObject, handTipLeftObject, handLeftObject, elbowLeftObject, shoulderLeftObject, spineShoulderObject, spineMidObject, spineBaseObject, hipRightObject, kneeRightObject, hipLeftObject, kneeLeftObject, footLeftObject, footRightObject
};
bagOfBonesDictionary = new Dictionary <long, List <OverlayObject> >();
ChestOrgansBehaviorDictionary = new Dictionary <long, List <Transform> >();
ArenaZ = arena.transform.lossyScale.z;
// print("bag of bpnes initialized");
foreach (OverlayObject oo in OverlayObjectsExample)
{
oo.BoneGameObject.transform.rotation = Quaternion.identity;
if (oo.ReferenceGameObjectTransform != null)
{
oo.InitialBoneLength = BoneLength(oo, oo.ReferenceGameObjectTransform.position);
}
else
{
oo.InitialBoneLength = 1;
}
}
}
private float BoneLength(OverlayObject oo, Vector3 endJointPos)
{
float boneLength = (oo.BoneGameObject.position - endJointPos).magnitude * boneScaleFactor;
return(boneLength);
}
private float FootBoneScale(OverlayObject oo)
{
return(oo.ReferenceGameObjectTransform.localScale.y);
}
void Update()
{
KinectManager kinectManager = KinectManager.Instance;
FacetrackingManager facetrackingManager = FacetrackingManager.Instance;
kinectManager.computeColorMap = true;
if (kinectManager && kinectManager.IsInitialized())
{
// get the kinect video stream
//Texture2D kinectColorTex = kinectManager.GetUsersClrTex();
RemoveSkeletonsForUnusedIDs(kinectManager);
// adding userIDs to the USerID list
if (kinectManager.IsUserDetected())
{
for (int i = 0; i < kinectManager.GetUsersCount(); i++)
{
bool found = false;
long ID = kinectManager.GetUserIdByIndex(i);
for (int j = 0; j < userIDs.Count; j++)
{
if (ID == userIDs[j])
{
found = true;
}
}
if (!found)
{
userIDs.Add(ID);
}
}
// add bones for any new users
for (int i = 0; i < userIDs.Count; i++)
{
long id = userIDs[i];
if (!bagOfBonesDictionary.ContainsKey(id) || !ChestOrgansBehaviorDictionary.ContainsKey(id)) // for each new user make a boneslist associated with his or her id.
{
if (bagOfBonesDictionary.ContainsKey(id))
{
bagOfBonesDictionary.Remove(id); // if the chestorgans dictionary did not have a list for this user, but the bones dictionary did, remove the bonesdictionary list to start with a new bones list, so that both dictionaries use child objects of the same skeleton.
}
List <OverlayObject> newUserBonesList = new List <OverlayObject>();
Transform thisUserSkeleton;
thisUserSkeleton = (Transform)Instantiate(SkeletonFBX);
CreateUserSkeleton(thisUserSkeleton, id);
foreach (OverlayObject oo in OverlayObjectsExample)
{
foreach (Transform child in thisUserSkeleton)
{
if (oo.BoneGameObject.gameObject.name == child.gameObject.name) // this user's skeleton bone matches bone in example bonelist
{
OverlayObject boneOverlayObject = new OverlayObject()
{
TrackedJointType = oo.TrackedJointType, BoneGameObject = child, InitialBoneLength = oo.InitialBoneLength
}; // uses initial values from example bones
if (oo.ReferenceGameObjectTransform != null)
{
boneOverlayObject.ReferenceGameObjectTransform = FindReferenceJointInThisSkeleton(oo.ReferenceGameObjectTransform.gameObject.name, thisUserSkeleton.transform);
}
oo.InitialRotation = oo.BoneGameObject.transform.rotation;
oo.BoneGameObject.transform.localScale = new Vector3(1, 1, 1);
newUserBonesList.Add(boneOverlayObject);
}
}
}
bagOfBonesDictionary.Add(id, newUserBonesList);
if (ChestOrgansBehaviorDictionary.ContainsKey(id))
{
ChestOrgansBehaviorDictionary.Remove(id);
}
List <Transform> newUserChestOrgansList = new List <Transform>();
Transform spineMid = thisUserSkeleton.FindChild("SpineMid");
Transform heartBox1 = spineMid.FindChild("HeartBox1");
newUserChestOrgansList.Insert((int)ChestOrgansEnum.HeartBeat, heartBox1);
Transform heartBox2 = spineMid.FindChild("HeartBox2");
newUserChestOrgansList.Insert((int)ChestOrgansEnum.HeartSys, heartBox2);
Transform diaphragmBoneMid = spineMid.FindChild("Diaphragm Bone Mid");
newUserChestOrgansList.Insert((int)ChestOrgansEnum.DiaphragmUp, diaphragmBoneMid);
Transform diaphragmBoneOut = spineMid.FindChild("Diaphragm Bone Out");
newUserChestOrgansList.Insert((int)ChestOrgansEnum.DiaphragmOut, diaphragmBoneOut);
ChestOrgansBehaviorDictionary.Add(id, newUserChestOrgansList);
}
}
var jp = TransformBones(kinectManager, facetrackingManager);
HeartAndLungTransform();
}
}
}