// Move active avatar based on the input JointData
public void MovePerson(PoseData live_data)
{
if (live_data == null)
{
return;
}
// stickContainer needs to always be updated, because score calculation relies on it
switch (activeType)
{
case AvatarType.CUBE:
stickContainer.MovePerson(live_data);
cubeContainer.MovePerson(live_data);
break;
case AvatarType.STICK:
stickContainer.MovePerson(live_data);
break;
case AvatarType.ROBOT:
stickContainer.MovePerson(live_data);
robotContainer.MovePerson(live_data);
break;
case AvatarType.SMPL:
stickContainer.MovePerson(live_data);
smplContainer.MovePerson(live_data);
break;
}
MoveIndicators();
}
// Converts JSON string to PoseData pose
public static PoseData JSONstring2PoseData(string frame_json)
{
JointData[] joint_data_recorded_array = new JointData[(int)JointId.Count];
PoseData recorded_data = new PoseData {
data = { }
};
//Debug.Log(frame_json);
PoseDataJSON saved_joint_data = JsonUtility.FromJson <PoseDataJSON>(frame_json);
for (JointId jt = 0; jt < JointId.Count; jt++)
{
// play recording
JointDataJSON jd = saved_joint_data.data[(int)jt];
Vector3 v_saved = JsonUtility.FromJson <Vector3>(jd.position);
Quaternion r_saved = JsonUtility.FromJson <Quaternion>(jd.rotation);
var joint_data_live = new JointData
{
Position = v_saved,
Orientation = r_saved
};
joint_data_recorded_array[(int)jt] = joint_data_live;
}
recorded_data = new PoseData {
data = joint_data_recorded_array
};
return(recorded_data);
}
// Animates all teacher avatars based on the JointData provided
void AnimateTeacher(PoseData recorded_data)
{
foreach (AvatarContainer avatar in avatarListTeacher)
{
avatar.MovePerson(recorded_data);
}
}
// Animates all self avatars based on the JointData provided
void AnimateSelf(PoseData live_data)
{
// MovePerson() considers which container to move
foreach (AvatarContainer avatar in avatarListSelf)
{
avatar.MovePerson(live_data);
}
}
public void MovePerson(PoseData joint_data_list)
{
// Remove mirroring before applying pose and readd it afterwards
// Necesary because MoveRiggedAvatar function works in global coordinates
Vector3 prevScale = SubContainerObject.transform.localScale;
SubContainerObject.transform.localScale = new Vector3(Math.Abs(prevScale.x), prevScale.y, prevScale.z);
RiggingUtils.MoveRiggedAvatar(animator, absoluteOffsetMap, joint_data_list, CharacterRootTransform, OffsetY, OffsetZ);
SubContainerObject.transform.localScale = prevScale;
}
public void MovePerson(PoseData joint_data_list)
{
//Place cubes at position and orietation of joints
for (JointId jt = 0; jt < JointId.Count; jt++)
{
var joint = joint_data_list.data[(int)jt];
var pos = joint.Position;
var orientation = joint.Orientation;
var v = new Vector3(pos[0], -pos[1], pos[2]) * 0.004f;
var r = new Quaternion(orientation[0], orientation[1], orientation[2], orientation[3]);
var obj = cubeObjects[(int)jt];
obj.transform.localPosition = v;
obj.transform.localRotation = r;
}
}
private async void StartWebsocket()
{
// Contains code to instantiate a websocket to obtain pose data
// Web socket is currently used only for Kinect Alternative
websocket = new WebSocket(WS_ip);
websocket.OnOpen += () =>
{
Debug.Log("WS connection open!");
};
websocket.OnError += (e) =>
{
Debug.Log("Error! " + e);
};
websocket.OnClose += (e) =>
{
Debug.Log("WS connection closed!");
};
websocket.OnMessage += (bytes) =>
{
// If joint information is recieved, set poseLiveWS
var message = System.Text.Encoding.UTF8.GetString(bytes);
Debug.Log("WS message received: " + message);
var remote_joints = PoseDataUtils.DeserializeRJL(message);
Debug.Log(remote_joints);
poseLiveWS = PoseDataUtils.Remote2PoseData(remote_joints);
};
// TODO: if using websocket, make this work
// Keep sending messages at every 0.3s
//InvokeRepeating("SendWebSocketMessage", 0.0f, 0.3f);
await websocket.Connect();
}
// Converts Azure Kinect SDK BT Body to PoseData pose
public static PoseData Body2PoseData(Body body)
{
JointData[] joint_data_live_array = new JointData[(int)JointId.Count];
PoseData live_data = new PoseData {
data = { }
};
for (JointId jt = 0; jt < JointId.Count; jt++)
{
Microsoft.Azure.Kinect.BodyTracking.Joint stickJoint = body.Skeleton.GetJoint(jt);
var stickPos = stickJoint.Position;
var stickOrientation = stickJoint.Quaternion;
JointData joint_data_live = new JointData
{
Position = new Vector3(stickPos.X, stickPos.Y, stickPos.Z),
Orientation = new Quaternion(stickOrientation.X, stickOrientation.Y, stickOrientation.Z, stickOrientation.W)
};
joint_data_live_array[(int)jt] = joint_data_live;
}
live_data = new PoseData {
data = joint_data_live_array
};
return(live_data);
}
public void MovePerson(PoseData joint_data_list)
{
/************************************Joints**************************************/
JointData stickJoint = joint_data_list.data[5];
var stickPos = stickJoint.Position;
var stickOrientation = stickJoint.Orientation;
var stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
var stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftShoulderStick.transform.localPosition = stickV;
LeftShoulderStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[12];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightShoulderStick.transform.localPosition = stickV;
RightShoulderStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[18];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftHipStick.transform.localPosition = stickV;
LeftHipStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[22];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightHipStick.transform.localPosition = stickV;
RightHipStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[6];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftElbowStick.transform.localPosition = stickV;
LeftElbowStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[13];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightElbowStick.transform.localPosition = stickV;
RightElbowStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[7];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftWristStick.transform.localPosition = stickV;
LeftWristStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[14];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightWristStick.transform.localPosition = stickV;
RightWristStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[19];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftKneeStick.transform.localPosition = stickV;
LeftKneeStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[23];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightKneeStick.transform.localPosition = stickV;
RightKneeStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[20];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftAnkleStick.transform.localPosition = stickV;
LeftAnkleStick.transform.localRotation = stickR;
stickJoint = joint_data_list.data[24];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightAnkleStick.transform.localPosition = stickV;
RightAnkleStick.transform.localRotation = stickR;
/************************************Head**************************************/
stickJoint = joint_data_list.data[28];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftEye.transform.localPosition = stickV;
LeftEye.transform.localRotation = stickR;
LeftEye.transform.localScale = new Vector3(0.3f, 0.2f, 0.2f);
stickJoint = joint_data_list.data[30];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightEye.transform.localPosition = stickV;
RightEye.transform.localRotation = stickR;
RightEye.transform.localScale = new Vector3(0.3f, 0.2f, 0.2f);
stickJoint = joint_data_list.data[27];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
NoseStick.transform.localPosition = stickV;
NoseStick.transform.localRotation = stickR;
NoseStick.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f);
stickJoint = joint_data_list.data[29];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftEar.transform.localPosition = stickV;
LeftEar.transform.localRotation = stickR;
LeftEar.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f);
LeftEar.transform.LookAt(LeftShoulderStick.transform.position);
LeftEar.transform.Rotate(90, 0, 0);
stickJoint = joint_data_list.data[31];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightEar.transform.localPosition = stickV;
RightEar.transform.localRotation = stickR;
RightEar.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f);
RightEar.transform.LookAt(RightShoulderStick.transform.position);
RightEar.transform.Rotate(90, 0, 0);
stickJoint = joint_data_list.data[27];
stickPos = stickJoint.Position;
stickOrientation = stickJoint.Orientation;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.008f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
MouthStick.transform.localPosition = stickV;
MouthStick.transform.localRotation = stickR;
MouthStick.transform.localScale = new Vector3(0.2f, 0.2f, 0.2f);
/************************************Body**************************************/
stickJoint = joint_data_list.data[5];
var stickJoint_b = joint_data_list.data[12];
stickPos = stickJoint.Position;
var stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
Vector3 stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
float stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
Shoulders.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
Shoulders.transform.LookAt(RightShoulderStick.transform.position);
Shoulders.transform.Rotate(90, 0, 0);
Shoulders.transform.localScale = new Vector3(0.2f, stick_length, 0.2f);
stickJoint = joint_data_list.data[18];
stickJoint_b = joint_data_list.data[22];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
HipStick.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
HipStick.transform.LookAt(RightHipStick.transform.position);
HipStick.transform.Rotate(90, 0, 0);
HipStick.transform.localScale = new Vector3(0.2f, stick_length, 0.2f);
stickJoint = joint_data_list.data[18];
stickJoint_b = joint_data_list.data[5];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
TorsoLeft.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
TorsoLeft.transform.LookAt(LeftShoulderStick.transform.position);
TorsoLeft.transform.Rotate(90, 0, 0);
TorsoLeft.transform.localScale = new Vector3(0.2f, stick_length, 0.2f);
stickJoint = joint_data_list.data[12];
stickJoint_b = joint_data_list.data[22];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
TorsoRight.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
TorsoRight.transform.LookAt(RightShoulderStick.transform.position);
TorsoRight.transform.Rotate(90, 0, 0);
TorsoRight.transform.localScale = new Vector3(0.2f, stick_length, 0.2f);
/************************************Arms**************************************/
stickJoint = joint_data_list.data[5];
stickJoint_b = joint_data_list.data[6];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftUpperArm.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
LeftUpperArm.transform.LookAt(LeftElbowStick.transform.position);
LeftUpperArm.transform.Rotate(90, 0, 0);
LeftUpperArm.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
stickJoint = joint_data_list.data[12];
stickJoint_b = joint_data_list.data[13];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightUpperArm.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
RightUpperArm.transform.LookAt(RightElbowStick.transform.position);
RightUpperArm.transform.Rotate(90, 0, 0);
RightUpperArm.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
stickJoint = joint_data_list.data[6];
stickJoint_b = joint_data_list.data[7];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftLowerArm.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
LeftLowerArm.transform.LookAt(LeftWristStick.transform.position);
LeftLowerArm.transform.Rotate(90, 0, 0);
LeftLowerArm.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
stickJoint = joint_data_list.data[13];
stickJoint_b = joint_data_list.data[14];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightLowerArm.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
RightLowerArm.transform.LookAt(RightWristStick.transform.position);
RightLowerArm.transform.Rotate(90, 0, 0);
RightLowerArm.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
/************************************Legs**************************************/
stickJoint = joint_data_list.data[18];
stickJoint_b = joint_data_list.data[19];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftUpperLeg.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
LeftUpperLeg.transform.LookAt(LeftHipStick.transform.position);
LeftUpperLeg.transform.Rotate(90, 0, 0);
LeftUpperLeg.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
stickJoint = joint_data_list.data[22];
stickJoint_b = joint_data_list.data[23];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightUpperLeg.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
RightUpperLeg.transform.LookAt(RightHipStick.transform.position);
RightUpperLeg.transform.Rotate(90, 0, 0);
RightUpperLeg.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
stickJoint = joint_data_list.data[19];
stickJoint_b = joint_data_list.data[20];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
LeftLowerLeg.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
LeftLowerLeg.transform.LookAt(LeftKneeStick.transform.position);
LeftLowerLeg.transform.Rotate(90, 0, 0);
LeftLowerLeg.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
stickJoint = joint_data_list.data[23];
stickJoint_b = joint_data_list.data[24];
stickPos = stickJoint.Position;
stickPos_b = stickJoint_b.Position;
stickOrientation = stickJoint.Orientation;
stick = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) - new Vector3(stickPos_b[0], -stickPos_b[1], stickPos_b[2]);
stick_length = stick.magnitude * 0.002f;
stickV = new Vector3(stickPos[0], -stickPos[1], stickPos[2]) * 0.004f;
stickR = new Quaternion(stickOrientation[0], stickOrientation[1], stickOrientation[2], stickOrientation[3]);
RightLowerLeg.transform.localPosition = new Vector3((stickPos[0] + stickPos_b[0]) * 0.5f, (-stickPos[1] - stickPos_b[1]) * 0.5f, (stickPos[2] + stickPos_b[2]) * 0.5f) * 0.008f;
RightLowerLeg.transform.LookAt(RightKneeStick.transform.position);
RightLowerLeg.transform.Rotate(90, 0, 0);
RightLowerLeg.transform.localScale = new Vector3(0.2f, 1.2f, 0.2f);
}
public static void MoveRiggedAvatar(Animator animator, Dictionary <JointId, Quaternion> absoluteOffsetMap, PoseData poseData, Transform CharacterRootTransform, float OffsetY, float OffsetZ, float scale = 0.001f)
{
for (int j = 0; j < (int)JointId.Count; j++)
{
if (MapKinectJoint((JointId)j) != HumanBodyBones.LastBone && absoluteOffsetMap.ContainsKey((JointId)j))
{
// get the absolute offset
Quaternion absOffset = absoluteOffsetMap[(JointId)j];
Transform finalJoint = animator.GetBoneTransform(MapKinectJoint((JointId)j));
Quaternion jointRot = AbsoluteJointRotations(poseData.data[j].Orientation, (JointId)j);
finalJoint.rotation = absOffset * Quaternion.Inverse(absOffset) * jointRot * absOffset;
if (j == 0)
{
// character root plus translation reading from the kinect, plus the offset from the script public variables
Vector3 root_data_pos = poseData.data[j].Position * scale;
finalJoint.position = CharacterRootTransform.position + new Vector3(root_data_pos.x, root_data_pos.y + OffsetY, root_data_pos.z - OffsetZ);
}
}
}
}
// Converts RemoteJointList pose to PoseData pose
public static PoseData Remote2PoseData(RemoteJointList rjl)
{
// format in lightweight-human-pose-estimation-3d-demo.pytorch
// Lightweight human pose estimation (https://github.com/Daniil-Osokin/lightweight-human-pose-estimation-3d-demo.pytorch) (Apache-2.0 License)
/*
* 0: Neck
* 1: Nose
* 2: BodyCenter(center of hips)
* 3: lShoulder
* 4: lElbow
* 5: lWrist,
* 6: lHip
* 7: lKnee
* 8: lAnkle
* 9: rShoulder
* 10: rElbow
* 11: rWrist
* 12: rHip
* 13: rKnee
* 14: rAnkle
* 15: rEye
* 16: lEye
* 17: rEar
* 18: lEar
*/
// format in Kinect Body Tracking SDK
// HipLeft 18 KneeLeft 19 AnkleLeft 20
// HipRight 22 KneeRight 23 AnkleRight 24
// ShoulderLeft 5
// ShoulderRight 12
// HipLeft 18 HipRight 22
// ElbowLeft 6 WristLeft 7
// ElbowRight 13 WristRight 14
// LeftEye 28
// RightEye 30
// Neck 3
// Nose 27
// Left Ear 29
// Right Ear 31
IDictionary <int, int> dict = new Dictionary <int, int>();
// map kinect keys to Pytorch keys
// in total 31 kinect and 18 Pytorch keys
dict.Add(18, 6);
dict.Add(19, 7);
dict.Add(20, 8);
dict.Add(22, 12);
dict.Add(23, 13);
dict.Add(24, 14);
dict.Add(5, 3);
dict.Add(12, 9);
dict.Add(6, 4);
dict.Add(7, 5);
dict.Add(13, 10);
dict.Add(14, 11);
dict.Add(28, 16);
dict.Add(30, 15);
dict.Add(3, 0);
dict.Add(27, 1);
dict.Add(29, 18);
dict.Add(31, 17);
JointData[] joint_data_received_array = new JointData[(int)JointId.Count];
Debug.Log(JsonUtility.ToJson(rjl));
List <List <double> > joint_data = rjl.values;
Debug.Log(joint_data);
float scaling = 10.0f;
for (JointId jt = 0; jt < JointId.Count; jt++)
{
if (dict.ContainsKey((int)jt))
{
int pytorch_index = dict[(int)jt];
List <double> jtd = joint_data[pytorch_index];
Debug.Log(jtd);
Vector3 v_rcv = new Vector3(scaling * (float)jtd[0], scaling * (float)jtd[1], scaling * (float)jtd[2]);
Quaternion r_rcv = new Quaternion(0, 0, 0, 0);
var joint_data_live = new JointData
{
Position = v_rcv,
Orientation = r_rcv
};
joint_data_received_array[(int)jt] = joint_data_live;
}
else
{ // just fill empty value as placeholder
Vector3 v_rcv = new Vector3(0, 0, 0);
Quaternion r_rcv = new Quaternion(0, 0, 0, 0);
var joint_data_live = new JointData
{
Position = v_rcv,
Orientation = r_rcv
};
joint_data_received_array[(int)jt] = joint_data_live;
}
}
PoseData received_data = new PoseData {
data = joint_data_received_array
};
return(received_data);
}
public PoseData GetNextPose()
{
switch (CurrentPoseInputSource)
{
case PoseInputSource.WEBSOCKET:
#if !UNITY_WEBGL || UNITY_EDITOR
websocket.DispatchMessageQueue();
#endif
// poseLiveWS is non-null if alternative is sending pose data over websocket
if (poseLiveWS != null)
{
// Assign last pose from websocket
CurrentPose = poseLiveWS;
}
else
{
Debug.Log("No pose recieved from WebSocket!");
}
break;
case PoseInputSource.FILE:
if (SequenceEnum != null && SequenceEnum.MoveNext())
{
_CurrentFilePoseNumber++;
}
else
{
// Quick and dirty way to loop (by reloading file)
if (SequenceEnum != null && !loop)
{
break;
}
LoadData();
SequenceEnum.MoveNext();
_CurrentFilePoseNumber = 1;
}
string frame_json = SequenceEnum.Current;
PoseData fake_live_data = PoseDataUtils.JSONstring2PoseData(frame_json);
CurrentPose = fake_live_data;
if (recording) // recording
{
File.AppendAllText(WriteDataPath, frame_json + Environment.NewLine);
}
break;
case PoseInputSource.KINECT:
if (device != null)
{
using (Capture capture = device.GetCapture())
{
// Make tracker estimate body
tracker.EnqueueCapture(capture);
// Code for getting RGB image from camera
Microsoft.Azure.Kinect.Sensor.Image color = capture.Color;
if (color != null && color.WidthPixels > 0 && (streamCanvas != null || videoRenderer != null))
{
UnityEngine.Object.Destroy(tex); // required to not keep old images in memory
tex = new Texture2D(color.WidthPixels, color.HeightPixels, TextureFormat.BGRA32, false);
tex.LoadRawTextureData(color.Memory.ToArray());
tex.Apply();
//Fetch the RawImage component from the GameObject
if (tex != null)
{
if (streamCanvas != null)
{
m_RawImage = streamCanvas.GetComponent <RawImage>();
m_RawImage.texture = tex;
}
if (videoRenderer != null)
{
videoRenderer.material.mainTexture = tex;
}
}
}
}
// Get pose estimate from tracker
using (Frame frame = tracker.PopResult())
{
//Debug.LogFormat("{0} bodies found.", frame.NumberOfBodies);
// At least one body found by Body Tracking
if (frame.NumberOfBodies > 0)
{
// Use first estimated person, if mutiple are in the image
// !!! There are (probably) no guarantees on consisitent ordering between estimates
//var bodies = frame.Bodies;
var body = frame.GetBody(0);
// Apply pose to user avatar(s)
PoseData live_data = PoseDataUtils.Body2PoseData(body);
if (recording) // recording
{
PoseDataJSON jdl = PoseDataUtils.Body2PoseDataJSON(body);
AppendRecordedFrame(jdl);
}
CurrentPose = live_data;
}
}
}
else
{
Debug.Log("device is null!");
}
break;
}
return(CurrentPose);
}