public void UpdateBody()
{
if (mKinect != null)
{
MultiSourceFrame frame = mFrameReader.AcquireLatestFrame();
if (frame != null)
{
using (var bodyFrame = frame.BodyFrameReference.AcquireFrame())
{
if (bodyFrame != null)
{
if (mSkeletons == null)
{
mSkeletons = new Microsoft.Kinect.Body[mKinect.BodyFrameSource.BodyCount];
mRightHandPoints = new DepthSpacePoint[mKinect.BodyFrameSource.BodyCount];
mLeftHandPoints = new DepthSpacePoint[mKinect.BodyFrameSource.BodyCount];
}
bodyFrame.GetAndRefreshBodyData(mSkeletons);
}
}
}
// Update hand positions
if (mSkeletons != null &&
mRightHandPoints != null &&
mLeftHandPoints != null)
{
for (int i = 0; i < mKinect.BodyFrameSource.BodyCount; i++)
{
mRightHandPoints[i] = mKinect.CoordinateMapper.MapCameraPointToDepthSpace(mSkeletons[i].Joints[JointType.HandRight].Position);
mLeftHandPoints[i] = mKinect.CoordinateMapper.MapCameraPointToDepthSpace(mSkeletons[i].Joints[JointType.HandLeft].Position);
}
}
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
var bodyIndexFrame = frame.BodyIndexFrameReference.AcquireFrame();
if (depthFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
//TextureScale.Bilinear(_ColorTexture, 512, 424);
depthFrame.CopyFrameDataToArray(_DepthData);
bodyIndexFrame.CopyFrameDataToArray(_BodyIndexData);
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
//_Mapper.MapDepthFrameToCameraSpace( DepthData_full, Camera3DPoints_full );
depthFrame.CopyFrameDataToArray(_DepthData);
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
}
public void ReadFrame()
{
if (SourceReaderRef != null)
{
//フレームの読み出し
var frame = SourceReaderRef.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(ColorData, ColorImageFormat.Bgra);
Marshal.Copy(ColorData, 0, ColorFrameMat.Data, ColorData.Length);
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
//var colorFrame = frame.ColorFrameReference.AcquireFrame();
//if (colorFrame != null)
//{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
//colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
//_ColorTexture.LoadRawTextureData(_ColorData);
//_ColorTexture.Apply();
depthFrame.CopyFrameDataToArray(_DepthData);
depthFrame.Dispose();
depthFrame = null;
}
//colorFrame.Dispose();
//colorFrame = null;
//}
frame = null;
}
}
}
void Update()
{
if (_Reader != null)
{
// ich bin mir nicht sicher ob es notwendig
// oder überhaupt sinvoll ist die einzelnen If's zu verschachteln
// dies ist entnommen aus der demo die depth und color in einem objekt angewendet hat
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
// begin color
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
// begin body
var bodyFrame = frame.BodyFrameReference.AcquireFrame();
if (bodyFrame != null)
{
if (_BodyData == null)
{
_BodyData = new Body[_Sensor.BodyFrameSource.BodyCount];
}
bodyFrame.GetAndRefreshBodyData(_BodyData);
// pause body
// begin depth
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
depthFrame.CopyFrameDataToArray(_DepthData);
depthFrame.Dispose();
depthFrame = null;
}
// end depth
// contnue body
bodyFrame.Dispose();
bodyFrame = null;
// end body
// continue color
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
_ColorTexture.LoadRawTextureData(_ColorData);
//_FaceColorTexture = CropedTexture(_ColorTexture);
//_FaceColorTexture.Apply();
_ColorTexture.Apply();
}
colorFrame.Dispose();
colorFrame = null;
// end color
}
frame = null;
}
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
isNewFrame = true;
// Update body data
var bodyFrame = frame.BodyFrameReference.AcquireFrame();
if (bodyFrame != null)
{
floorClipPlane = bodyFrame.FloorClipPlane;
if (_BodyData == null)
{
_BodyData = new Body[_Sensor.BodyFrameSource.BodyCount];
}
bodyFrame.GetAndRefreshBodyData(_BodyData);
bodyFrame.Dispose();
bodyFrame = null;
}
// Update depth data
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
if (_DepthData == null)
{
_DepthData = new ushort[_Sensor.DepthFrameSource.FrameDescription.LengthInPixels];
}
depthFrame.CopyFrameDataToArray(_DepthData);
depthFrame.Dispose();
depthFrame = null;
}
// Update body index data
var bodyIndexFrame = frame.BodyIndexFrameReference.AcquireFrame();
if (bodyIndexFrame != null)
{
if (_BodyIndexData == null)
{
_BodyIndexData = new byte[_Sensor.BodyIndexFrameSource.FrameDescription.LengthInPixels];
}
bodyIndexFrame.CopyFrameDataToArray(_BodyIndexData);
bodyIndexFrame.Dispose();
bodyIndexFrame = null;
}
frame = null;
}
else
{
isNewFrame = false;
}
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
// Populate arrays - Save all required data
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
depthFrame.CopyFrameDataToArray(_DepthData);
// Map to color from depth frame
_Mapper.MapDepthFrameToColorSpace(_DepthData, _ColorPoints);
// Clip the depth frame
EvaluateDepthData();
// Clip the color frame
EvaluateColorData();
// Get chosen depth data from initial depth data.
DSPDepthData();
// Get chosen color data from initial color data.
DSPColorData();
// Apply Color Texture
_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
// Apply Clipped Color Texture
_ClippedColorTexture.LoadRawTextureData(_ClippedColorData);
_ClippedColorTexture.Apply();
// Apply Chosen Color Texture
_ChosenColorTexture.LoadRawTextureData(_ChosenColorData);
_ChosenColorTexture.Apply();
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
}
public bool GetMultiSourceFrame(KinectInterop.SensorData sensorData)
{
if (multiSourceFrameReader != null)
{
multiSourceFrame = multiSourceFrameReader.AcquireLatestFrame();
return(multiSourceFrame != null);
}
return(false);
}
public void PollMostRecentColorFrame()
{
MultiSourceFrame multiFrame = _reader.AcquireLatestFrame();
if (multiFrame == null)
{
return;
}
using (ColorFrame frame = multiFrame.ColorFrameReference.AcquireFrame())
{
if (frame == null)
{
return; // Could not find multi-frame or color-frame
}
frame.CopyConvertedFrameDataToArray(MostRecentColorFrame, ColorImageFormat.Rgba);
}
}
void Update()
{
// Get FPS
elapsedCounter += Time.deltaTime;
if (elapsedCounter > 1.0)
{
fps = frameCount / elapsedCounter;
frameCount = 0;
elapsedCounter = 0.0;
}
if (_multiSourceFrameReader == null)
{
return;
}
var pMultiSourceFrame = _multiSourceFrameReader.AcquireLatestFrame();
if (pMultiSourceFrame != null)
{
frameCount++;
nullFrame = false;
using (var pDepthFrame = pMultiSourceFrame.DepthFrameReference.AcquireFrame())
{
using (var pColorFrame = pMultiSourceFrame.ColorFrameReference.AcquireFrame())
{
using (var pBodyIndexFrame = pMultiSourceFrame.BodyIndexFrameReference.AcquireFrame())
{
// Get Depth Frame Data.
if (pDepthFrame != null)
{
var pDepthData = GCHandle.Alloc(_depthBuffer, GCHandleType.Pinned);
pDepthFrame.CopyFrameDataToIntPtr(pDepthData.AddrOfPinnedObject(), (uint)_depthBuffer.Length * sizeof(ushort));
pDepthData.Free();
}
// Get Color Frame Data
if (pColorFrame != null)
{
var pColorData = GCHandle.Alloc(_colorBuffer, GCHandleType.Pinned);
pColorFrame.CopyConvertedFrameDataToIntPtr(pColorData.AddrOfPinnedObject(), (uint)_colorBuffer.Length, ColorImageFormat.Rgba);
pColorData.Free();
}
}
}
}
ProcessFrame();
}
else
{
nullFrame = true;
}
}
void Update()
{
if (reader != null)
{
var frame = reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
var depthFrame = frame.DepthFrameReference.AcquireFrame();
var bodyFrame = frame.BodyFrameReference.AcquireFrame();
if (colorFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(colorData, ColorImageFormat.Rgba);
colorFrame.Dispose();
colorBuffer.SetData(colorData);
}
if (depthFrame != null)
{
depthFrame.CopyFrameDataToArray(depthData);
depthFrame.Dispose();
kinect.CoordinateMapper.MapDepthFrameToColorSpace(depthData, colorSpacePoints);
kinect.CoordinateMapper.MapDepthFrameToCameraSpace(depthData, cameraSpacePoints);
colorSpacePointBuffer.SetData(colorSpacePoints);
cameraSpacePointBuffer.SetData(cameraSpacePoints);
}
if (bodyFrame != null)
{
var temp = floorClipPlane;
floorClipPlane = bodyFrame.FloorClipPlane;
if (floorClipPlane.W == 0)
{
floorClipPlane.W = temp.W;
}
var kinectRot = Quaternion.FromToRotation(new Vector3(floorClipPlane.X, floorClipPlane.Y, floorClipPlane.Z), Vector3.up);
var kinectHeight = floorClipPlane.W;
pointCloudCS.SetVector("_ResetRot", new UnityEngine.Vector4(kinectRot.x, kinectRot.y, kinectRot.z, kinectRot.w));
pointCloudCS.SetFloat("_KinectHeight", kinectHeight);
bodyFrame.Dispose();
}
}
var kernel = pointCloudCS.FindKernel("buildPointCloud");
pointCloudCS.SetBuffer(kernel, "_ColorData", colorBuffer);
pointCloudCS.SetBuffer(kernel, "_ColorSpacePointData", colorSpacePointBuffer);
pointCloudCS.SetBuffer(kernel, "_CameraSpacePointData", cameraSpacePointBuffer);
pointCloudCS.SetBuffer(kernel, "_PointCloudData", pointCloudBuffer);
pointCloudCS.Dispatch(kernel, depthDataLength / 8, 1, 1);
}
}
void Update()
{
// Get FPS
// elapsedCounter+=Time.deltaTime;
// if(elapsedCounter > 1.0)
// {
// fps = frameCount / elapsedCounter;
// frameCount = 0;
// elapsedCounter = 0.0;
// }
if (m_pMultiSourceFrameReader == null)
{
return;
}
var pMultiSourceFrame = m_pMultiSourceFrameReader.AcquireLatestFrame();
if (pMultiSourceFrame != null)
{
// frameCount++;
nullFrame = false;
using (var pDepthFrame = pMultiSourceFrame.DepthFrameReference.AcquireFrame())
{
using (var pBodyIndexFrame = pMultiSourceFrame.BodyIndexFrameReference.AcquireFrame())
{
// Get Depth Frame Data.
if (pDepthFrame != null)
{
var pDepthData = GCHandle.Alloc(pDepthBuffer, GCHandleType.Pinned);
pDepthFrame.CopyFrameDataToIntPtr(pDepthData.AddrOfPinnedObject(), (uint)pDepthBuffer.Length * sizeof(ushort));
pDepthData.Free();
}
// Get BodyIndex Frame Data.
if (pBodyIndexFrame != null)
{
var pBodyIndexData = GCHandle.Alloc(pBodyIndexBuffer, GCHandleType.Pinned);
pBodyIndexFrame.CopyFrameDataToIntPtr(pBodyIndexData.AddrOfPinnedObject(), (uint)pBodyIndexBuffer.Length);
pBodyIndexData.Free();
}
}
}
ProcessFrame();
}
else
{
nullFrame = true;
}
}
// Update is called once per frame
void Update()
{
if (reader != null)
{
MultiSourceFrame MSFrame = reader.AcquireLatestFrame();
if (MSFrame != null)
{
using (DepthFrame frame = MSFrame.DepthFrameReference.AcquireFrame())
{
if (frame != null)
{
frame.CopyFrameDataToArray(depthData);
coordinateMapper.MapDepthFrameToCameraSpace(depthData, cameraSpacePoints);
// Debug.Log(cameraSpacePoints.Length);
}
}
MSFrame = null;
}
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
if (useColor)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
colorFrame.Dispose();
colorFrame = null;
}
}
if (useDepth)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
depthFrame.CopyFrameDataToArray(_DepthData);
depthFrame.Dispose();
depthFrame = null;
mapper.MapDepthFrameToCameraSpace(_DepthData, _RealWorldPoints);
}
}
if (useBodyIndex)
{
var bodyIndexFrame = frame.BodyIndexFrameReference.AcquireFrame();
if (bodyIndexFrame != null)
{
bodyIndexFrame.CopyFrameDataToArray(_BodyIndexData);
bodyIndexFrame.Dispose();
bodyIndexFrame = null;
}
}
frame = null;
}
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
var bodyFrame = frame.BodyFrameReference.AcquireFrame();
if (bodyFrame != null)
{
if (_BodyData == null)
{
_BodyData = new Body[_Sensor.BodyFrameSource.BodyCount];
}
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
depthFrame.CopyFrameDataToArray(_DepthData);
ProcessFrame();
bodyFrame.GetAndRefreshBodyData(_BodyData);
bodyFrame.Dispose();
frame = null;
}
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
}
public void UpdateMultiFrame()
{
if (_Sensor == null)
{
return;
}
var frame = _Reader.AcquireLatestFrame();
if (frame == null)
{
return;
}
UpdateDepthFrame(frame.DepthFrameReference.AcquireFrame());
UpdateBodyFrame(frame.BodyFrameReference.AcquireFrame());
}
public Tuple <Point3DCollection, Point3DCollection> GetDepthDataFromLatestFrame()
{
int i = 0;
MultiSourceFrame frame = null;
while (frame == null || i < 1000)
{
frame = multiSourceFrameReader.AcquireLatestFrame();
i++;
}
if (frame != null)
{
return(GetDepthData(frame));
}
return(null);
}
void Reader_MultiSourceFrameArrived(object sender, MultiSourceFrameArrivedEventArgs e)
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
var bodyIndexFrame = frame.BodyIndexFrameReference.AcquireFrame();
if (bodyIndexFrame != null)
{
//heavy cpu usage... rethink colour input
//colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
//_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
depthFrame.CopyFrameDataToArray(_DepthData);
bodyIndexFrame.CopyFrameDataToArray(_BodyIndexData);
bodyIndexFrame.Dispose();
bodyIndexFrame = null;
}
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
}
IEnumerator FrameUpdate()
{
do
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
// _ColorTexture.LoadRawTextureData(_ColorData);
// _ColorTexture.Apply();
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
isFresh = true;
depthFrame.CopyFrameDataToArray(_DepthData);
//CreateDepthTexture(depthFrame);
//_DepthTexture.LoadRawTextureData(_DepthDataBytes);
//_DepthTexture.Apply();
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
yield return(new WaitForSeconds(0.01f));
} while (true);
}
//When the timer ticks the colorframe is polled and checked if there is a face
private void OnTimerTick(object sender, ElapsedEventArgs e)
{
MultiSourceFrame multiSourceFrame = _multiSourceFrameReader.AcquireLatestFrame();
if (multiSourceFrame != null)
{
using (ColorFrame colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame())
{
if (colorFrame != null)
{
if (!_isEngaged)
{
NewEngagmentCheck(colorFrame);
}
else
{
StopEngagmentCheck(colorFrame);
if (_audioSource.IsRecording())
{
CheckStopRecording(colorFrame);
}
else
{
CheckStartRecording(colorFrame);
}
SendFaces(colorFrame);
}
}
else if (_isEngaged) //no color frame -> if engaged check not engagment
{
StopEngagmentCheck(null);
}
}
}
else if (_isEngaged) //no frame -> if engaged check not engagment
{
StopEngagmentCheck(null);
}
}
void Update()
{
if (m_pMultiSourceFrameReader == null)
{
return;
}
MultiSourceFrame pMultiSourceFrame = m_pMultiSourceFrameReader.AcquireLatestFrame();
if (pMultiSourceFrame != null)
{
frameCount++;
using (DepthFrame pDepthFrame = pMultiSourceFrame.DepthFrameReference.AcquireFrame())
{
using (ColorFrame pColorFrame = pMultiSourceFrame.ColorFrameReference.AcquireFrame())
{
// Get Depth Frame Data
if (pDepthFrame != null)
{
GCHandle pDepthData = GCHandle.Alloc(pDepthBuffer, GCHandleType.Pinned);
pDepthFrame.CopyFrameDataToIntPtr(pDepthData.AddrOfPinnedObject(), (uint)pDepthBuffer.Length * sizeof(ushort));
pDepthData.Free();
pDepthFrame.Dispose();
}
// Get Color Frame Data
if (pColorFrame != null)
{
GCHandle pColorData = GCHandle.Alloc(pColorBuffer, GCHandleType.Pinned);
pColorFrame.CopyConvertedFrameDataToIntPtr(pColorData.AddrOfPinnedObject(), (uint)pColorBuffer.Length, ColorImageFormat.Rgba);
pColorData.Free();
pColorFrame.Dispose();
}
ProcessFrame();
}
}
}
}
private void Update()
{
if (frameReader != null)
{
MultiSourceFrame multiSourceFrame = frameReader.AcquireLatestFrame();
if (multiSourceFrame != null)
{
BodyFrame bodyFrame = multiSourceFrame.BodyFrameReference.AcquireFrame();
if (bodyFrame != null)
{
if (bodyData == null)
{
bodyData = new Body[kinectSensor.BodyFrameSource.BodyCount];
}
bodyFrame.GetAndRefreshBodyData(bodyData);
floor = new Floor(bodyFrame.FloorClipPlane);
bodyFrame.Dispose();
bodyFrame = null;
}
DetectCenterBody();
multiSourceFrame = null;
isNewFrame = true;
}
else
{
isNewFrame = false;
}
}
}
public override void Update()
{
trackedBodyJoints.Clear();
if (frameReader != null)
{
var frame = frameReader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
colorFrame.Dispose();
colorFrame = null;
}
}
frame = null;
}
}
void Update()
{
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
Debug.Log("Frame not null");
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(_ColorData, ColorImageFormat.Rgba);
_ColorTexture.LoadRawTextureData(_ColorData);
_ColorTexture.Apply();
depthFrame.CopyFrameDataToArray(_DepthData);
depthFrame.Dispose();
depthFrame = null;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
else
{
Debug.Log("Frame null !");
}
}
}
void Update()
{
//局部变量
int depthWidth = 0;
int depthHeight = 0;
int colorWidth = 0;
int colorHeight = 0;
int bodyIndexWidth = 0;
int bodyIndexHeight = 0;
bool multiSourceFrameProcessed = false;
bool colorFrameProcessed = false;
bool depthFrameProcessed = false;
bool bodyIndexFrameProcessed = false;
bool infraredFrameProcessed = false;
#region _multiSourceReader Begin Work
if (_multiSourceReader != null)
{
MultiSourceFrame multiSourceFrame = _multiSourceReader.AcquireLatestFrame();
if (multiSourceFrame != null)
{
using (DepthFrame depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame())
{
using (ColorFrame colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame())
{
using (BodyIndexFrame bodyIndexFrame = multiSourceFrame.BodyIndexFrameReference.AcquireFrame())
{
using (InfraredFrame infraredFrame = multiSourceFrame.InfraredFrameReference.AcquireFrame())
{
if (depthFrame != null)
{
FrameDescription depthFrameDescription = depthFrame.FrameDescription;
depthWidth = depthFrameDescription.Width;
depthHeight = depthFrameDescription.Height;
if ((depthWidth * depthHeight) == this.depthFrameData.Length)
{
depthFrame.CopyFrameDataToArray(this.depthFrameData);
depthFrameProcessed = true;
}
}
if (colorFrame != null)
{
FrameDescription colorFrameDescription = colorFrame.FrameDescription;
colorWidth = colorFrameDescription.Width;
colorHeight = colorFrameDescription.Height;
if ((colorWidth * colorHeight * this.bytesPerPixel) == this.colorFrameData.Length)
{
if (colorFrame.RawColorImageFormat == ColorImageFormat.Rgba)
{
colorFrame.CopyRawFrameDataToArray(this.colorFrameData);
}
else
{
colorFrame.CopyConvertedFrameDataToArray(this.colorFrameData, ColorImageFormat.Rgba);
_kinectColorTexture.LoadRawTextureData(colorFrameData);
_kinectColorTexture.Apply();
}
colorFrameProcessed = true;
}
}
if (bodyIndexFrame != null)
{
FrameDescription bodyIndexFrameDescription = bodyIndexFrame.FrameDescription;
bodyIndexWidth = bodyIndexFrameDescription.Width;
bodyIndexHeight = bodyIndexFrameDescription.Height;
if ((bodyIndexWidth * bodyIndexHeight) == this.bodyIndexFrameData.Length)
{
bodyIndexFrame.CopyFrameDataToArray(this.bodyIndexFrameData);
bodyIndexFrameProcessed = true;
}
}
if (infraredFrame != null)
{
Debug.Log("AAA");
infraredFrame.CopyFrameDataToArray(_kinectInfraredData);
int index = 0;
foreach (var ir in _kinectInfraredData)
{
byte intensity = (byte)(ir >> 8);
_kinectInfraredRawData[index++] = intensity;
_kinectInfraredRawData[index++] = intensity;
_kinectInfraredRawData[index++] = intensity;
_kinectInfraredRawData[index++] = 255; // Alpha
}
_kinectInfraredTexture.LoadRawTextureData(_kinectInfraredRawData);
_kinectInfraredTexture.Apply();
infraredFrameProcessed = true;
}
multiSourceFrameProcessed = true;
}
}
}
}
}
// we got all frames
if (multiSourceFrameProcessed && depthFrameProcessed && colorFrameProcessed && bodyIndexFrameProcessed && infraredFrameProcessed)
{
_coordinateMapper.MapDepthFrameToColorSpace(this.depthFrameData, this.colorPoints);
Array.Clear(this.displayPixels, 0, this.displayPixels.Length);
// loop over each row and column of the depth
for (int y = 0; y < depthHeight; ++y)
{
for (int x = 0; x < depthWidth; ++x)
{
// calculate index into depth array
int depthIndex = (y * depthWidth) + x;
byte player = this.bodyIndexFrameData[depthIndex];
// if we're tracking a player for the current pixel, sets its color and alpha to full
if (player != 0xff)
{
// retrieve the depth to color mapping for the current depth pixel
ColorSpacePoint colorPoint = this.colorPoints[depthIndex];
// make sure the depth pixel maps to a valid point in color space
int colorX = (int)Math.Floor(colorPoint.X + 0.5);
int colorY = (int)Math.Floor(colorPoint.Y + 0.5);
if ((colorX >= 0) && (colorX < colorWidth) && (colorY >= 0) && (colorY < colorHeight))
{
// calculate index into color array
int colorIndex = ((colorY * colorWidth) + colorX) * this.bytesPerPixel;
// set source for copy to the color pixel
int displayIndex = depthIndex * this.bytesPerPixel;
// write out blue byte
this.displayPixels[displayIndex++] = this.colorFrameData[colorIndex++];
// write out green byte
this.displayPixels[displayIndex++] = this.colorFrameData[colorIndex++];
// write out red byte
this.displayPixels[displayIndex++] = this.colorFrameData[colorIndex];
// write out alpha byte
this.displayPixels[displayIndex] = 0xff;
}
}
}
}
}
}
#endregion
DisPlayerTexture();
}
/// <summary>
/// Updates the frame by grabbing the rgb and depth from the Kinnect.
/// </summary>
/// <returns><c>true</c>, if frame was successfully updated, <c>false</c> otherwise.</returns>
/// <param name="state">State.</param>
public bool UpdateFrame(HoloProxies.Engine.trackerState state)
{
bool success = false;
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
//Debug.Log( "in here1" );
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
//Debug.Log( "in here2" );
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
//Debug.Log( "in here3" );
// get color data + texture
colorFrame.CopyConvertedFrameDataToArray(ColorData_full, ColorImageFormat.Rgba);
// get depth data
depthFrame.CopyFrameDataToArray(DepthData_full);
// Create a depth texture
//int index = 0;
//foreach (var ir in DepthData_full)
//{
// byte intensity = (byte)(ir >> 8);
// DepthRaw[index++] = intensity;
// DepthRaw[index++] = intensity;
// DepthRaw[index++] = intensity;
// DepthRaw[index++] = 255; // Alpha
//}
//DepthTexture_full.LoadRawTextureData( DepthRaw );
//DepthTexture_full.Apply(); TODO fix this
// dispose frame
depthFrame.Dispose();
depthFrame = null;
// Map depth to RGBD
AlignRGBD();
// Unproject to 3D points in camera space (based on bounding box)
UnprojectFrom2Dto3D();
// Downsample all data frames if necessary
DownsampleAndFilterRGBDImage();
// Apply downsampled texture
ColorTexture.LoadRawTextureData(ColorData);
ColorTextureVisual.LoadRawTextureData(ColorData);
// Unproject points to 3D space
PreparePointCloud(state);
if (plotHistogram)
{
for (int i = 0; i < HistogramTexture.height; i++)
{
for (int j = 0; j < HistogramTexture.width; j++)
{
int idx = i * HistogramTexture.width + j;
HistogramTexture.SetPixel(j, i, new Color(histogram.posterior[idx * 256], 0, 0));
}
}
HistogramTexture.Apply();
}
success = true;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
return(success);
}
void Update()
{
if (reader != null)
{
var frame = reader.AcquireLatestFrame();
if (frame != null)
{
var colorFrame = frame.ColorFrameReference.AcquireFrame();
var depthFrame = frame.DepthFrameReference.AcquireFrame();
var bodyFrame = frame.BodyFrameReference.AcquireFrame();
var bodyIndexFrame = frame.BodyIndexFrameReference.AcquireFrame();
if (colorFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(colorData, ColorImageFormat.Rgba);
colorFrame.Dispose();
colorTex.LoadRawTextureData(colorData);
colorTex.Apply();
}
if (depthFrame != null)
{
depthFrame.CopyFrameDataToArray(depthData);
depthFrame.Dispose();
kinect.CoordinateMapper.MapDepthFrameToColorSpace(depthData, colorSpacePoints);
kinect.CoordinateMapper.MapDepthFrameToCameraSpace(depthData, cameraSpacePoints);
colorSpacePointBuffer.SetData(colorSpacePoints);
cameraSpacePointBuffer.SetData(cameraSpacePoints);
}
if (bodyFrame != null)
{
var temp = floorClipPlane;
floorClipPlane = bodyFrame.FloorClipPlane;
if (floorClipPlane.W == 0)
{
floorClipPlane.W = temp.W;
}
var kinectRot = Quaternion.FromToRotation(new Vector3(floorClipPlane.X, floorClipPlane.Y, floorClipPlane.Z), Vector3.up);
var kinectHeight = floorClipPlane.W;
if (!useKinectRot)
{
kinectRot = Quaternion.identity;
kinectHeight = 0f;
}
pointCloudCS.SetVector("_ResetRot", new UnityEngine.Vector4(kinectRot.x, kinectRot.y, kinectRot.z, kinectRot.w));
pointCloudCS.SetFloat("_KinectHeight", kinectHeight);
bodyFrame.GetAndRefreshBodyData(bodyData);
for (var i = 0; i < bodyData.Length; i++)
{
kinectBodies[i].SetBodyData(bodyData[i], kinectRot, kinectHeight);
}
bodyFrame.Dispose();
if (cameraToRender != null)
{
cameraToRender.transform.position = Vector3.up * kinectHeight;
cameraToRender.transform.rotation = kinectRot;
}
}
if (bodyIndexFrame != null)
{
bodyIndexFrame.CopyFrameDataToArray(bodyIndexData);
bodyIndexFrame.Dispose();
bodyIndexTex.LoadRawTextureData(bodyIndexData);
bodyIndexTex.Apply();
}
}
var kernel = pointCloudCS.FindKernel("buildVertex");
pointCloudCS.SetBuffer(kernel, "_ColorSpacePointData", colorSpacePointBuffer);
pointCloudCS.SetBuffer(kernel, "_CameraSpacePointData", cameraSpacePointBuffer);
pointCloudCS.SetBuffer(kernel, "_VertexDataBuffer", vertexBuffer);
pointCloudCS.Dispatch(kernel, depthDataLength / 8, 1, 1);
}
}
void Update()
{
//////// FPS - BEGIN
elapsedCounter += Time.deltaTime;
if (elapsedCounter > 1.0)
{
fps = frameCount / elapsedCounter;
frameCount = 0;
elapsedCounter = 0.0;
}
//////// FPS - END
if (!init)
{
return;
}
var pMultiSourceFrame = multiSourceFrameReader.AcquireLatestFrame();
if (pMultiSourceFrame != null)
{
//////// FPS - BEGIN
frameCount++;
//////// FPS - END
//////// NULL_FRAME - BEGIN
// nullFrame = false;
//////// NULL_FRAME - END
using (var pDepthFrame = pMultiSourceFrame.DepthFrameReference.AcquireFrame()) {
if (pDepthFrame != null)
{
var pDepthData = GCHandle.Alloc(pDepthBuffer, GCHandleType.Pinned);
pDepthFrame.CopyFrameDataToIntPtr(pDepthData.AddrOfPinnedObject(), (uint)pDepthBuffer.Length * sizeof(ushort));
pDepthData.Free();
}
}
using (var pColorFrame = pMultiSourceFrame.ColorFrameReference.AcquireFrame()) {
if (pColorFrame != null)
{
var pColorData = GCHandle.Alloc(pColorBuffer, GCHandleType.Pinned);
pColorFrame.CopyConvertedFrameDataToIntPtr(pColorData.AddrOfPinnedObject(), (uint)pColorBuffer.Length, ColorImageFormat.Bgra);
pColorData.Free();
}
}
using (var pBodyIndexFrame = pMultiSourceFrame.BodyIndexFrameReference.AcquireFrame()) {
if (pBodyIndexFrame != null)
{
var pBodyIndexData = GCHandle.Alloc(pBodyIndexBuffer, GCHandleType.Pinned);
pBodyIndexFrame.CopyFrameDataToIntPtr(pBodyIndexData.AddrOfPinnedObject(), (uint)pBodyIndexBuffer.Length);
pBodyIndexData.Free();
}
}
ProcessFrame();
}
//////// NULL_FRAME - BEGIN
/*
* else {
* nullFrame = true;
* }
*/
//////// NULL_FRAME - END
}
void Update()
{
if (multiSourceReader != null)
{
var frame = multiSourceReader.AcquireLatestFrame();
if (frame != null)
{
// update fps
var time = Time.time;
var interval = time - lastFrameTime;
frameInterval = Mathf.Lerp(frameInterval, interval, 0.1f);
fps = 1f / frameInterval;
lastFrameTime = time;
using (var depthFrame = frame.DepthFrameReference.AcquireFrame())
{
if (depthFrame != null)
{
depthFrame.CopyFrameDataToArray(depthFrameData);
int depthFrameWidth = depthFrame.FrameDescription.Width;
int depthFrameHeight = depthFrame.FrameDescription.Height;
int depthFramePixelCount = depthFrameWidth * depthFrameHeight;
sensor.CoordinateMapper.MapDepthFrameToCameraSpace(depthFrameData, cameraSpacePoints);
mapDimensions[0] = depthFrameWidth;
mapDimensions[1] = depthFrameHeight;
if (tempPositionTexture != null && (tempPositionTexture.width != depthFrameWidth || tempPositionTexture.height != depthFrameHeight))
{
Destroy(tempPositionTexture);
tempPositionTexture = null;
}
if (positionBuffer != null && positionBuffer.count != cameraSpacePoints.Length)
{
positionBuffer.Dispose();
positionBuffer = null;
}
if (tempPositionTexture == null)
{
tempPositionTexture = new RenderTexture(depthFrameWidth, depthFrameHeight, 0, RenderTextureFormat.ARGBHalf);
tempPositionTexture.enableRandomWrite = true;
tempPositionTexture.Create();
}
if (positionBuffer == null)
{
positionBuffer = new ComputeBuffer(cameraSpacePoints.Length, sizeof(float) * 3);
}
positionBuffer.SetData(cameraSpacePoints);
PointCloudBaker.SetInts("MapDimensions", mapDimensions);
PointCloudBaker.SetBuffer(0, "PositionBuffer", positionBuffer);
PointCloudBaker.SetTexture(0, "PositionTexture", tempPositionTexture);
PointCloudBaker.Dispatch(0, depthFrameWidth / 8, depthFrameHeight / 8, 1);
Graphics.CopyTexture(tempPositionTexture, PointCloudMap);
if (UseColor)
{
using (var colorFrame = frame.ColorFrameReference.AcquireFrame())
{
if (colorFrame != null)
{
int colorFrameWidth = colorFrame.FrameDescription.Width;
int colorFrameHeight = colorFrame.FrameDescription.Height;
if (colorSourceTexture != null && (colorSourceTexture.width != colorFrameWidth || colorSourceTexture.height != colorFrameHeight))
{
Destroy(colorSourceTexture);
colorSourceTexture = null;
}
if (colorSourceTexture == null)
{
colorSourceTexture = new Texture2D(colorFrameWidth, colorFrameHeight, TextureFormat.RGBA32, false);
}
if (tempColorTexture != null && (tempColorTexture.width != depthFrameWidth || tempColorTexture.height != depthFrameHeight))
{
Destroy(tempColorTexture);
tempColorTexture = null;
}
if (tempColorTexture == null)
{
tempColorTexture = new RenderTexture(depthFrameWidth, depthFrameHeight, 0, RenderTextureFormat.ARGB32);
tempColorTexture.enableRandomWrite = true;
tempColorTexture.Create();
}
if (depthToColorMapBuffer != null && depthToColorMapBuffer.count != depthFramePixelCount)
{
depthToColorMapBuffer.Dispose();
depthToColorMapBuffer = null;
}
if (depthToColorMapBuffer == null)
{
depthToColorMapBuffer = new ComputeBuffer(depthFramePixelCount, sizeof(float) * 2);
}
// Map depth points to color space
using (var depthBuffer = depthFrame.LockImageBuffer())
{
var depthToColorPointsPtr = GCHandle.Alloc(depthToColorPoints, GCHandleType.Pinned);
sensor.CoordinateMapper.MapDepthFrameToColorSpaceUsingIntPtr(
depthBuffer.UnderlyingBuffer,
(int)sensor.DepthFrameSource.FrameDescription.LengthInPixels * sizeof(ushort),
depthToColorPointsPtr.AddrOfPinnedObject(),
sensor.DepthFrameSource.FrameDescription.LengthInPixels);// * sizeof(float) * 2);
depthToColorMapBuffer.SetData(depthToColorPointsPtr.AddrOfPinnedObject(), depthFramePixelCount, sizeof(float) * 2);
depthToColorPointsPtr.Free();
}
var colorDataPtr = GCHandle.Alloc(colorFrameData, GCHandleType.Pinned);
colorFrame.CopyConvertedFrameDataToIntPtr(colorDataPtr.AddrOfPinnedObject(), (uint)colorFrameData.Length, ColorImageFormat.Rgba);
colorSourceTexture.LoadRawTextureData(colorDataPtr.AddrOfPinnedObject(), colorFrameData.Length * sizeof(float) * 4);
colorSourceTexture.Apply();
colorDataPtr.Free();
int bakeColorKernel = PointCloudBaker.FindKernel("BakeColor");
PointCloudBaker.SetTexture(bakeColorKernel, "ColorSource", colorSourceTexture);
PointCloudBaker.SetBuffer(bakeColorKernel, "DepthToColorMap", depthToColorMapBuffer); // Upload mapped points
PointCloudBaker.SetTexture(bakeColorKernel, "ColorTexture", tempColorTexture);
PointCloudBaker.Dispatch(bakeColorKernel, depthFrameWidth / 8, depthFrameHeight / 8, 1);
Graphics.CopyTexture(tempColorTexture, ColorMap);
}
}
}
depthFrame.Dispose();
}
}
}
}
}
