public NiteGUI(NiteController niteController)
{
//depth_Thread = new Thread(new ThreadStart(this.updateusermap));
//image_Thread = new Thread(new ThreadStart(this.updatergbimage));
//depth_Thread.Start();
//image_Thread.Start();
this.niteController = niteController;
// // Setting the Kinect in certain angle
// this.startAngle = 2000;
// this.kinectDevice = new CLNUIDevice(startAngle);
//
// Init depth & label map related stuff
usersMapSize = NiteWrapper.GetDepthWidth() * NiteWrapper.GetDepthHeight();
usersLblTex = new Texture2D(NiteWrapper.GetDepthWidth(), NiteWrapper.GetDepthHeight());
usersMapColors = new Color[usersMapSize];
usersMapRect = new Rect(Screen.width - usersLblTex.width / 2, Screen.height - usersLblTex.height / 2, usersLblTex.width / 2, usersLblTex.height / 2);
usersLabelMap = new short[usersMapSize];
usersDepthMap = new short[usersMapSize];
usersHistogramMap = new float[5000];
// Init camera image stuff
RGBwidth = NiteWrapper.getRGBWidth();
RGBheight = NiteWrapper.getRGBHeight();
image = new short[RGBwidth * RGBheight * 3];
usersImageTex = new Texture2D(RGBwidth, RGBheight);
usersImageColors = new Color[usersMapSize];
usersImageRect = new Rect(Screen.width - usersImageTex.width / 2, Screen.height - usersImageTex.height / 2, usersImageTex.width / 2, usersImageTex.height / 2);
bg = GameObject.Find("Background Image").GetComponent <GUITexture>();
bg.pixelInset = new Rect(-Screen.width / 2, -Screen.height / 2, Screen.width, Screen.height);
stateIcon = Resources.Load("lookingUser") as Texture2D;
}
public void UpdateRgbImage()
{
// if (image_Thread.ThreadState == ThreadState.Suspended) {
// image_Thread.Resume();
// }
Marshal.Copy(NiteWrapper.getRGB(), image, 0, RGBwidth * RGBheight * 3);
int p = 0;
int flipIndex;
int usersMapSize = 640 * 480;
for (int i = 0; i < usersMapSize; i++)
{
flipIndex = usersMapSize - i - 1;
Color c = new Color((float)image[p++] / 255f, (float)image[p++] / 255f, (float)image[p++] / 255f);
usersImageColors[flipIndex] = c;
}
usersImageTex.SetPixels(usersImageColors);
usersImageTex.Apply();
bg.texture = usersImageTex;
double I = NiteWrapper.getIntensity(niteController.getCurrentUser());
total_intensity -= intensity[p_intensity];
total_intensity += I;
intensity[p_intensity] = I;
p_intensity = (p_intensity > 8)? 0 : p_intensity + 1;
Light light = GameObject.Find("Point light").GetComponent <Light>();
light.intensity = 1 * (float)(total_intensity / 10.0);
niteController.strIntensity = "user = "******", intensity = " + I;
}
void OnUserLost(uint UserId)
{
Debug.Log(String.Format("[{0}] User lost", UserId));
calibratedUser = false;
Debug.Log("Starting to look for users");
NiteWrapper.StartLookingForUsers(NewUser, CalibrationStarted, CalibrationFailed, CalibrationSuccess, UserLost);
gui.stateIcon = Resources.Load("lookingUser") as Texture2D;
}
public void Update()
{
//update the skeleton, depth-, and rgb image
NiteWrapper.Update(true);
if (!calibratedUser)
{
//gui.UpdateUserMap();
}
gui.UpdateRgbImage();
List <Rig> destroyedRigs = new List <Rig>();
//update the rigs controlled by this controller
foreach (Rig rig in registeredRigs)
{
Vector3 position;
Quaternion rotation;
// Check if rig still exists:
if (rig.riggedObject != null)
{
// Update body location
if (GetJointPosition(NiteWrapper.SkeletonJoint.TORSO_CENTER, out position))
{
rig.jointMapping[NiteWrapper.SkeletonJoint.TORSO_CENTER].position = position;
}
// Update the rotation for each joint
foreach (KeyValuePair <NiteWrapper.SkeletonJoint, Transform> pair in rig.jointMapping)
{
if (GetJointOrientation(pair.Key, out rotation))
{
pair.Value.rotation = rotation * rig.referenceOrientation[pair.Key];
}
}
// Update joint positions
foreach (KeyValuePair <NiteWrapper.SkeletonJoint, Transform> pair in rig.jointMapping)
{
if (GetJointPosition(pair.Key, out position))
{
pair.Value.position = position;
}
}
}
else
{
destroyedRigs.Add(rig);
}
}
foreach (Rig rig in destroyedRigs)
{
registeredRigs.Remove(rig);
}
}
void Update()
{
// Next NITE frame
NiteWrapper.Update(true);
// update the visual user map
UpdateUserMap();
// update avatars
foreach (KeyValuePair <uint, SoldierAvatar> pair in calibratedUsers)
{
pair.Value.UpdateAvatar(pair.Key);
}
}
public bool GetJointPosition(NiteWrapper.SkeletonJoint joint, out Vector3 position)
{
if (kinectConnect && calibratedUser)
{
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(calibratedUserId, joint, ref trans);
// Nite gives position in mm convert to Unity unit = meters
position = new Vector3(trans.pos.x / 1000.0F, trans.pos.y / 1000.0F, trans.pos.z / 1000.0F);
return(trans.pos.confidence > confidenceThreshold);
}
else
{
position = new Vector3(0.0F, 0.0F, 0.0F);
return(false);
}
}
public bool GetJointOrientation(NiteWrapper.SkeletonJoint joint, out Quaternion rotation)
{
if (kinectConnect && calibratedUser)
{
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(calibratedUserId, joint, ref trans);
Vector3 worldZVec = new Vector3(trans.ori.m02, trans.ori.m12, trans.ori.m22);
Vector3 worldYVec = new Vector3(trans.ori.m01, trans.ori.m11, trans.ori.m21);
rotation = Quaternion.LookRotation(worldZVec, worldYVec);
return(trans.ori.confidence > confidenceThreshold);
}
else
{
rotation = Quaternion.identity;
return(false);
}
}
void Start()
{
uint rc = NiteWrapper.Init(new StringBuilder(".\\OpenNI.xml"));
if (rc != 0)
{
Debug.Log(String.Format("Error initing OpenNI: {0}", Marshal.PtrToStringAnsi(NiteWrapper.GetStatusString(rc))));
}
// Init depth & label map related stuff
usersMapSize = NiteWrapper.GetDepthWidth() * NiteWrapper.GetDepthHeight();
usersLblTex = new Texture2D(NiteWrapper.GetDepthWidth(), NiteWrapper.GetDepthHeight());
usersMapColors = new Color[usersMapSize];
usersMapRect = new Rect(Screen.width - usersLblTex.width / 2, Screen.height - usersLblTex.height / 2, usersLblTex.width / 2, usersLblTex.height / 2);
usersLabelMap = new short[usersMapSize];
usersDepthMap = new short[usersMapSize];
usersHistogramMap = new float[5000];
// text
caption = GameObject.Find("GUI Text").guiText;
// init our avatar controllers
soldiers = new SoldierAvatar[2];
soldiers[0] = new SoldierAvatar(GameObject.Find("Soldier1"));
//soldiers[1] = new SoldierAvatar(GameObject.Find("Soldier2")); // This line was removed so we can have 1 soldier without errors
// init user lists - one will contain all users, the second will contain only calibrated & mapped users
allUsers = new List <uint>();
calibratedUsers = new Dictionary <uint, SoldierAvatar>();
// init user callbacks
NewUser = new NiteWrapper.UserDelegate(OnNewUser);
CalibrationStarted = new NiteWrapper.UserDelegate(OnCalibrationStarted);
CalibrationFailed = new NiteWrapper.UserDelegate(OnCalibrationFailed);
CalibrationSuccess = new NiteWrapper.UserDelegate(OnCalibrationSuccess);
UserLost = new NiteWrapper.UserDelegate(OnUserLost);
// Start looking
NiteWrapper.StartLookingForUsers(NewUser, CalibrationStarted, CalibrationFailed, CalibrationSuccess, UserLost);
Debug.Log("Waiting for users to calibrate");
// set default smoothing
NiteWrapper.SetSkeletonSmoothing(0.6);
}
void TransformBone(uint userId, NiteWrapper.SkeletonJoint joint, Transform dest)
{
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(userId, joint, ref trans);
// only modify joint if confidence is high enough in this frame
if (trans.ori.confidence > 0.5)
{
// Z coordinate in OpenNI is opposite from Unity. We will create a quat
// to rotate from OpenNI to Unity (relative to initial rotation)
Vector3 worldZVec = new Vector3(trans.ori.m02, -trans.ori.m12, trans.ori.m22);
Vector3 worldYVec = new Vector3(-trans.ori.m01, trans.ori.m11, -trans.ori.m21);
Quaternion jointRotation = Quaternion.LookRotation(worldZVec, worldYVec);
Quaternion newRotation = jointRotation * initialRotations[(int)joint];
// Some smoothing
dest.rotation = Quaternion.Slerp(dest.rotation, newRotation, Time.deltaTime * 20);
}
}
void OnCalibrationSuccess(uint UserId)
{
Debug.Log(String.Format("[{0}] Calibration success", UserId));
// Associate this user to an unused soldier avatar
for (int i = 0; i < soldiers.Length; i++)
{
if (!calibratedUsers.ContainsValue(soldiers[i]))
{
calibratedUsers.Add(UserId, soldiers[i]);
break;
}
}
// Should we stop looking for users?
if (calibratedUsers.Count == soldiers.Length)
{
Debug.Log("Stopping to look for users");
NiteWrapper.StopLookingForUsers();
}
}
void OnUserLost(uint UserId)
{
Debug.Log(String.Format("[{0}] User lost", UserId));
// If this was one of our calibrated users, mapped to an avatar
if (calibratedUsers.ContainsKey(UserId))
{
// reset the avatar and remove from list
calibratedUsers[UserId].RotateToCalibrationPose();
calibratedUsers.Remove(UserId);
// Should we start looking for users again?
if (calibratedUsers.Count < soldiers.Length)
{
Debug.Log("Starting to look for users");
NiteWrapper.StartLookingForUsers(NewUser, CalibrationStarted, CalibrationFailed, CalibrationSuccess, UserLost);
}
}
// remove from global users list
allUsers.Remove(UserId);
}
void OnGUI()
{
if (calibratedUsers.Count < soldiers.Length)
{
GUI.DrawTexture(usersMapRect, usersLblTex);
caption.text = String.Format("Waiting for {0} more users to calibrate...", soldiers.Length - calibratedUsers.Count);
}
else
{
caption.text = "All users calibrated";
}
foreach (uint userId in allUsers)
{
float progress = NiteWrapper.GetUserPausePoseProgress(userId);
if (NiteWrapper.GetUserPausePoseProgress(userId) > 0.0)
{
caption.text = String.Format("[User {0}] Pause pose progress: {1}", userId, progress);
break;
}
}
}
public NiteController(NewUserCallback onNewUser)
{
// initialize the Kinect
uint rc = NiteWrapper.Init(new StringBuilder(".\\OpenNI.xml"));
if (rc != 0)
{
Debug.Log(String.Format("Error initializing OpenNI: {0}", Marshal.PtrToStringAnsi(NiteWrapper.GetStatusString(rc))));
return;
}
else
{
kinectConnect = true;
}
// init user callbacks
NewUser = new NiteWrapper.UserDelegate(OnNewUser);
CalibrationStarted = new NiteWrapper.UserDelegate(OnCalibrationStarted);
CalibrationFailed = new NiteWrapper.UserDelegate(OnCalibrationFailed);
CalibrationSuccess = new NiteWrapper.UserDelegate(OnCalibrationSuccess);
UserLost = new NiteWrapper.UserDelegate(OnUserLost);
// Start looking
NiteWrapper.StartLookingForUsers(NewUser, CalibrationStarted, CalibrationFailed, CalibrationSuccess, UserLost);
Debug.Log("Waiting for users to calibrate");
// set default smoothing
NiteWrapper.SetSkeletonSmoothing(0.0);
// set new user callback
this.onNewUser = onNewUser;
// initialize gui
gui = new NiteGUI(this);
// initialize list of rigs to animate
registeredRigs = new List <Rig>();
}
// Rotate the spine of the avatar to the direction of the main fly arm.
void RotateSpineInFlight(uint userId, NiteWrapper.SkeletonJoint joint) {
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(userId, joint, ref trans);
// Smooth spine rotation, but currently results in a buggy avatar.
//spine.forward = Vector3.SmoothDamp(spine.forward, -mainFlyArm.right, ref velocity, smoothTime);
// Instant spine rotation to the direction of the main fly arm.
spine.forward = -mainFlyArm.right;
}
// This function moves the avatar around according to the movement of the user relative to the kinect.
void MoveBone(uint userId, NiteWrapper.SkeletonJoint joint, Transform dest) {
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(userId, joint, ref trans);
// Save the first position of the user relative to the kinect once it stopped flying.
if (firstMoveAfterFlight == true)
kinectOffsetPosition = new Vector3(trans.pos.x/1000, trans.pos.y/1000 -1, -trans.pos.z/1000);
// The position of the avatar is based on the characterPosition (i.e. the position where the avatar
// has flown to) and the movement around the initial position of the user after flight (kinectOffsetPosition)
// relative to the kinect.
dest.position = characterPosition - kinectOffsetPosition + new Vector3(trans.pos.x/1000, trans.pos.y/1000 -1, -trans.pos.z/1000);
}
// Retrieves the latest rotation information from the kinect.
void RotateBone(uint userId, NiteWrapper.SkeletonJoint joint, Transform dest) {
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(userId, joint, ref trans);
// Only modify joint if confidence is high enough in this frame.
// We set it to 0 since we always want the latest rotation information.
if (trans.ori.confidence > 0.0) {
// Z coordinate in OpenNI is opposite from Unity. We will create a quaternion
// to rotate from OpenNI to Unity (relative to initial rotation).
Vector3 worldZVec = new Vector3(-trans.ori.m02, -trans.ori.m12, trans.ori.m22);
Vector3 worldYVec = new Vector3(trans.ori.m01, trans.ori.m11, -trans.ori.m21);
Quaternion jointRotation = Quaternion.LookRotation(worldZVec, worldYVec);
Quaternion newRotation = jointRotation * initialRotations[(int)joint];
// Smoothing results in a shaky avatar, but might be useful if it is made to work.
// dest.rotation = Quaternion.Slerp(dest.rotation, newRotation, Time.deltaTime * 20);
dest.rotation = newRotation;
}
}
public bool GetJointPosition(NiteWrapper.SkeletonJoint joint, out Vector3 position) {
if (kinectConnect && calibratedUser) {
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(calibratedUserId, joint, ref trans);
// Nite gives position in mm convert to Unity unit = meters
position = new Vector3(trans.pos.x/1000.0F, trans.pos.y/1000.0F, trans.pos.z/1000.0F);
return (trans.pos.confidence > confidenceThreshold);
}
else {
position = new Vector3(0.0F, 0.0F, 0.0F);
return false;
}
}
public bool GetJointOrientation(NiteWrapper.SkeletonJoint joint, out Quaternion rotation) {
if (kinectConnect && calibratedUser) {
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(calibratedUserId, joint, ref trans);
Vector3 worldZVec = new Vector3(trans.ori.m02, trans.ori.m12, trans.ori.m22);
Vector3 worldYVec = new Vector3(trans.ori.m01, trans.ori.m11, trans.ori.m21);
rotation = Quaternion.LookRotation(worldZVec, worldYVec);
return (trans.ori.confidence > confidenceThreshold);
}
else {
rotation = Quaternion.identity;
return false;
}
}
void UpdateUserMap()
{
// copy over the maps
Marshal.Copy(NiteWrapper.GetUsersLabelMap(), usersLabelMap, 0, usersMapSize);
Marshal.Copy(NiteWrapper.GetUsersDepthMap(), usersDepthMap, 0, usersMapSize);
// we will be flipping the texture as we convert label map to color array
int flipIndex, i;
int numOfPoints = 0;
Array.Clear(usersHistogramMap, 0, usersHistogramMap.Length);
// calculate cumulative histogram for depth
for (i = 0; i < usersMapSize; i++)
{
// only calculate for depth that contains users
if (usersLabelMap[i] != 0)
{
usersHistogramMap[usersDepthMap[i]]++;
numOfPoints++;
}
}
if (numOfPoints > 0)
{
for (i = 1; i < usersHistogramMap.Length; i++)
{
usersHistogramMap[i] += usersHistogramMap[i - 1];
}
for (i = 0; i < usersHistogramMap.Length; i++)
{
usersHistogramMap[i] = 1.0f - (usersHistogramMap[i] / numOfPoints);
}
}
// create the actual users texture based on label map and depth histogram
for (i = 0; i < usersMapSize; i++)
{
flipIndex = usersMapSize - i - 1;
if (usersLabelMap[i] == 0)
{
usersMapColors[flipIndex] = Color.clear;
}
else
{
// create a blending color based on the depth histogram
Color c = new Color(usersHistogramMap[usersDepthMap[i]], usersHistogramMap[usersDepthMap[i]], usersHistogramMap[usersDepthMap[i]], 0.9f);
switch (usersLabelMap[i] % 4)
{
case 0:
usersMapColors[flipIndex] = Color.red * c;
break;
case 1:
usersMapColors[flipIndex] = Color.green * c;
break;
case 2:
usersMapColors[flipIndex] = Color.blue * c;
break;
case 3:
usersMapColors[flipIndex] = Color.magenta * c;
break;
}
}
}
usersLblTex.SetPixels(usersMapColors);
usersLblTex.Apply();
}
void OnApplicationQuit()
{
NiteWrapper.Shutdown();
}
void OnCalibrationSuccess(uint UserId)
{
Debug.Log(String.Format("[{0}] Calibration success", UserId));
calibratedUser = true;
calibratedUserId = UserId;
//Rotate Kinect so center of mass is in the middle
//CLNUIDevice device = gui.kinectDevice;
//device.RotateToCenter(UserId);
Debug.Log("Stopping to look for users");
NiteWrapper.StopLookingForUsers();
int size = 0;
int length = 0;
double[] girth = new double[16];
for (int i = 0; i < 16; i++)
{
IntPtr raw_data = NiteWrapper.getFatness((int)UserId, i, ref size, ref length, ref girth[i]);
int[] data = new int[size];
Marshal.Copy(raw_data, data, 0, size);
sizeData[i] = data;
diameter[i] = (float)length / 1000.0F; //convert from mm to m
}
double[] chest_g = { girth[12], girth[13], girth[14], girth[15] };
bool done = false;
while (!done)
{
done = true;
for (int i = 1; i < 4; i++)
{
if (chest_g[i - 1] > chest_g[i])
{
double temp = chest_g[i];
chest_g[i] = chest_g[i - 1];
chest_g[i - 1] = temp;
done = false;
}
}
}
double ChestGirth = chest_g[0] / 10;
if (ChestGirth < 70)
{
clothSize = ChestGirth + " is too small!";
}
else if (ChestGirth < 78)
{
clothSize = "XXS : extra extra small (" + ChestGirth + ")";
}
else if (ChestGirth < 86)
{
clothSize = "XS : extra small (" + ChestGirth + ")";
}
else if (ChestGirth < 94)
{
clothSize = "S : small (" + ChestGirth + ")";
}
else if (ChestGirth < 102)
{
clothSize = "M : medium (" + ChestGirth + ")";
}
else if (ChestGirth < 110)
{
clothSize = "L : large (" + ChestGirth + ")";
}
else if (ChestGirth < 118)
{
clothSize = "XL : extra large (" + ChestGirth + ")";
}
else if (ChestGirth < 129)
{
clothSize = "XXL : extra extra large (" + ChestGirth + ")";
}
else if (ChestGirth < 141)
{
clothSize = "3XL : extra extra extra large (" + ChestGirth + ")";
}
else
{
clothSize = "too large! (" + ChestGirth + ")";
}
Debug.Log(clothSize);
this.onNewUser();
gui.stateIcon = Resources.Load("calibrated") as Texture2D;
}
public Vector3 left, right; // 3d world positions of hands
public HandPositions (NiteWrapper.SkeletonJointTransformation leftHand, NiteWrapper.SkeletonJointTransformation rightHand) {
left = new Vector3(leftHand.pos.x, leftHand.pos.y, leftHand.pos.z);
right = new Vector3(rightHand.pos.x, rightHand.pos.y, rightHand.pos.z);
}
void TransformBone(uint userId, NiteWrapper.SkeletonJoint joint, Transform dest)
{
NiteWrapper.SkeletonJointTransformation trans = new NiteWrapper.SkeletonJointTransformation();
NiteWrapper.GetJointTransformation(userId, joint, ref trans);
// only modify joint if confidence is high enough in this frame
if (trans.ori.confidence > 0.5)
{
// Z coordinate in OpenNI is opposite from Unity. We will create a quat
// to rotate from OpenNI to Unity (relative to initial rotation)
Vector3 worldZVec = new Vector3(-trans.ori.m02, -trans.ori.m12, trans.ori.m22);
Vector3 worldYVec = new Vector3(trans.ori.m01, trans.ori.m11, -trans.ori.m21);
Quaternion jointRotation = Quaternion.LookRotation(worldZVec, worldYVec);
Quaternion newRotation = jointRotation * initialRotations[(int)joint];
// Some smoothing
dest.rotation = Quaternion.Slerp(dest.rotation, newRotation, Time.deltaTime * 20);
}
}
