private int NuiSkeletonGetNextFrame(uint dwMillisecondsToWait, ref NuiSkeletonFrame pSkeletonFrame)
{
if (sourceFlags != KinectInterop.FrameSource.TypeAudio)
{
// non-audio sources
uint iFrameLength = (uint)GetSkeletonFrameLength();
bool bSuccess = GetSkeletonFrameData(ref pSkeletonFrame, ref iFrameLength, true);
return(bSuccess ? 0 : -1);
}
else
{
// audio only
int hr = GetNextSkeletonFrame(dwMillisecondsToWait);
if (hr == 0)
{
uint iFrameLength = (uint)GetSkeletonFrameLength();
bool bSuccess = GetSkeletonFrameData(ref pSkeletonFrame, ref iFrameLength, true);
return(bSuccess ? 0 : -1);
}
return(hr);
}
}
public static bool PollSkeleton(NuiTransformSmoothParameters smoothParameters, ref NuiSkeletonFrame skeletonFrame)
{
bool newSkeleton = false;
int hr = KinectWrapper.NuiSkeletonGetNextFrame(0, ref skeletonFrame);
if (hr == 0)
{
newSkeleton = true;
}
if (newSkeleton)
{
hr = KinectWrapper.NuiTransformSmooth(ref skeletonFrame, ref smoothParameters);
if (hr != 0)
{
Debug.Log("Skeleton Data Smoothing failed");
}
}
return(newSkeleton);
}
public static uint NuiSkeletonGetNextFrame(uint dwMillisecondsToWait, ref NuiSkeletonFrame pSkeletonFrame)
{
//TODO: NuiSkeletonGetNextFrame
if (dwMillisecondsToWait != 0)
{
Debug.LogWarning("non-zero wdMillisecondsToWait is not implemented, will be ignored.");
}
RawFrameData rawFrame = KinectProxy.Instance.GetLatestFrame();
if (rawFrame != null)
{
int count = 0;
foreach (var body in rawFrame.bodies)
{
PopulateSkeletonData(body, ref pSkeletonFrame.SkeletonData[count++]);
}
for (; count < Constants.SkeletonCount; count++)
{
pSkeletonFrame.SkeletonData[count].eTrackingState = NuiSkeletonTrackingState.NotTracked;
}
return(0);
}
return((uint)NuiErrorCodes.SkeletalEngineBusy);
}
public static bool PollSkeleton(ref NuiTransformSmoothParameters smoothParameters, ref NuiSkeletonFrame skeletonFrame)
{
bool newSkeleton = false;
int hr = KinectWrapper.NuiSkeletonGetNextFrame(0, ref skeletonFrame);
if(hr == 0)
{
newSkeleton = true;
}
if(newSkeleton)
{
hr = KinectWrapper.NuiTransformSmooth(ref skeletonFrame, ref smoothParameters);
if(hr != 0)
{
Debug.Log("Skeleton Data Smoothing failed");
}
}
return newSkeleton;
}
public static extern int NuiTransformSmooth(ref NuiSkeletonFrame pSkeletonFrame, ref NuiTransformSmoothParameters pSmoothingParams);
public static int NuiSkeletonGetNextFrame(uint dwMillisecondsToWait, ref NuiSkeletonFrame pSkeletonFrame)
{
int hr = GetNextSkeletonFrame(dwMillisecondsToWait);
if(hr == 0)
{
uint iFrameLength = (uint)GetSkeletonFrameLength();
bool bSuccess = GetSkeletonFrameData(ref pSkeletonFrame, ref iFrameLength, true);
return bSuccess ? 0 : -1;
}
return hr;
}
public static extern int NuiSkeletonGetNextFrame(uint dwMillisecondsToWait, ref NuiSkeletonFrame pSkeletonFrame);
public static extern bool GetSkeletonFrameData(ref NuiSkeletonFrame pSkeletonData, ref uint iDataBufLen, bool bNewFrame);
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
if((dwFlags & KinectInterop.FrameSource.TypeAudio) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesAudio;
}
FacetrackingManager[] faceManagers = GameObject.FindObjectsOfType(typeof(FacetrackingManager)) as FacetrackingManager[];
if(faceManagers != null && faceManagers.Length > 0)
{
for(int i = 0; i < faceManagers.Length; i++)
{
if(faceManagers[i].enabled)
{
//Debug.Log("Found FacetrackingManager => UsesColor");
nuiFlags |= NuiInitializeFlags.UsesColor;
break;
}
}
}
SpeechManager[] speechManagers = GameObject.FindObjectsOfType(typeof(SpeechManager)) as SpeechManager[];
if(speechManagers != null && speechManagers.Length > 0)
{
for(int i = 0; i < speechManagers.Length; i++)
{
if(speechManagers[i].enabled)
{
//Debug.Log("Found SpeechManager => UsesAudio");
nuiFlags |= NuiInitializeFlags.UsesAudio;
break;
}
}
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if(hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if(hr != 0)
{
Debug.LogError(string.Format("Error initializing KinectInteraction: hr=0x{0:X}", hr));
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
sensorData.depthCameraFOV = 46.6f;
sensorData.colorCameraFOV = 48.6f;
sensorData.depthCameraOffset = 0.01f;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return sensorData;
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return null;
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if (hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if (hr != 0)
{
Debug.LogError("Initialization of KinectInteraction failed.");
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return(sensorData);
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return(null);
}
public KinectInterop.SensorData OpenDefaultSensor (KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if(hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if(hr != 0)
{
Debug.LogError("Initialization of KinectInteraction failed.");
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return sensorData;
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return null;
}
public static extern int NuiTransformSmooth(ref NuiSkeletonFrame pSkeletonFrame, ref NuiTransformSmoothParameters pSmoothingParams);
public static extern int NuiSkeletonGetNextFrame(uint dwMillisecondsToWait, ref NuiSkeletonFrame pSkeletonFrame);
private void processSkeletonFromFrame(NuiSkeletonFrame frame)
{
int[] tracked = new int[Kinect.Constants.NuiSkeletonMaxTracked];
tracked[0] = -1;
tracked[1] = -1;
int trackedCount = 0;
//update players
for (int ii = 0; ii < Kinect.Constants.NuiSkeletonCount; ii++)
{
players[ii] = frame.SkeletonData[ii].eTrackingState;
if (players[ii] == Kinect.NuiSkeletonTrackingState.SkeletonTracked)
{
tracked[trackedCount] = ii;
trackedCount++;
}
}
//this should really use trackingID instead of index, but for now this is fine
switch (trackedCount)
{
case 0:
//Felix
//Make kinect assets invisible if no player is tracked and no replay is running
Player.SetFocus(Player.Replaying);
trackedPlayers[0] = -1;
trackedPlayers[1] = -1;
break;
case 1:
//last frame there were no players: assign new player to p1
if (!Player.Focus)
{
Player.SetFocus(true);
}
if (trackedPlayers[0] < 0 && trackedPlayers[1] < 0)
{
trackedPlayers[0] = tracked[0];
}
//last frame there was one player, keep that player in the same spot
else if (trackedPlayers[0] < 0)
{
trackedPlayers[1] = tracked[0];
}
else if (trackedPlayers[1] < 0)
{
trackedPlayers[0] = tracked[0];
}
//there were two players, keep the one with the same index (if possible)
else
{
if (tracked[0] == trackedPlayers[0])
{
trackedPlayers[1] = -1;
}
else if (tracked[0] == trackedPlayers[1])
{
trackedPlayers[0] = -1;
}
else
{
trackedPlayers[0] = tracked[0];
trackedPlayers[1] = -1;
}
}
break;
case 2:
//last frame there were no players: assign new players to p1 and p2
if (trackedPlayers[0] < 0 && trackedPlayers[1] < 0)
{
trackedPlayers[0] = tracked[0];
trackedPlayers[1] = tracked[1];
}
//last frame there was one player, keep that player in the same spot
else if (trackedPlayers[0] < 0)
{
if (trackedPlayers[1] == tracked[0])
{
trackedPlayers[0] = tracked[1];
}
else
{
trackedPlayers[0] = tracked[0];
trackedPlayers[1] = tracked[1];
}
}
else if (trackedPlayers[1] < 0)
{
if (trackedPlayers[0] == tracked[1])
{
trackedPlayers[1] = tracked[0];
}
else
{
trackedPlayers[0] = tracked[0];
trackedPlayers[1] = tracked[1];
}
}
//there were two players, keep the one with the same index (if possible)
else
{
if (trackedPlayers[0] == tracked[1] || trackedPlayers[1] == tracked[0])
{
trackedPlayers[0] = tracked[1];
trackedPlayers[1] = tracked[0];
}
else
{
trackedPlayers[0] = tracked[0];
trackedPlayers[1] = tracked[1];
}
}
break;
}
//update the bone positions, velocities, and tracking states)
for (int player = 0; player < 2; player++)
{
//print(player + ", " +trackedPlayers[player]);
if (trackedPlayers[player] >= 0)
{
for (int bone = 0; bone < (int)Kinect.NuiSkeletonPositionIndex.Count; bone++)
{
Vector3 oldpos = bonePos[player, bone];
bonePos[player, bone] = kinectToWorld.MultiplyPoint3x4(frame.SkeletonData[trackedPlayers[player]].SkeletonPositions[bone]);
//bonePos[player,bone] = kinectToWorld.MultiplyPoint3x4(bonePos[player, bone]);
rawBonePos[player, bone] = frame.SkeletonData[trackedPlayers[player]].SkeletonPositions[bone];
Kinect.NuiSkeletonBoneOrientation[] or = kinect.getBoneOrientations(frame.SkeletonData[trackedPlayers[player]]);
boneLocalOrientation[player, bone] = or[bone].hierarchicalRotation.rotationQuaternion.GetQuaternion();
boneAbsoluteOrientation[player, bone] = or[bone].absoluteRotation.rotationQuaternion.GetQuaternion();
//print("index " + bone + ", start" + (int)or[bone].startJoint + ", end" + (int)or[bone].endJoint);
boneVel[player, bone] = (bonePos[player, bone] - oldpos) / deltaTime;
boneState[player, bone] = frame.SkeletonData[trackedPlayers[player]].eSkeletonPositionTrackingState[bone];
//print(kinect.getSkeleton().SkeletonData[player].Position.z);
}
}
}
}
private int NuiSkeletonGetNextFrame(uint dwMillisecondsToWait, ref NuiSkeletonFrame pSkeletonFrame)
{
if(sourceFlags != KinectInterop.FrameSource.TypeAudio)
{
// non-audio sources
uint iFrameLength = (uint)GetSkeletonFrameLength();
bool bSuccess = GetSkeletonFrameData(ref pSkeletonFrame, ref iFrameLength, true);
return bSuccess ? 0 : -1;
}
else
{
// audio only
int hr = GetNextSkeletonFrame(dwMillisecondsToWait);
if(hr == 0)
{
uint iFrameLength = (uint)GetSkeletonFrameLength();
bool bSuccess = GetSkeletonFrameData(ref pSkeletonFrame, ref iFrameLength, true);
return bSuccess ? 0 : -1;
}
return hr;
}
}
public static int NuiTransformSmooth(ref NuiSkeletonFrame pSkeletonFrame, ref NuiTransformSmoothParameters pSmoothingParams)
{
return(0);
}
private static extern bool GetSkeletonFrameData(ref NuiSkeletonFrame pSkeletonData, ref uint iDataBufLen, bool bNewFrame);