/// <summary>
/// Gets 3D points [units = meters] and calculated pf vector
/// </summary>
/// <param name="state">State.</param>
private void PreparePointCloud(HoloProxies.Engine.trackerState state)
{
boundingbox = findBoundingBoxFromCurrentState(state);
for (int i = 0; i < Height; i++)
{
for (int j = 0; j < Width; j++)
{
int idx = i * Width + j;
// if point is not in the bounding box
if (j < boundingbox.x || j >= boundingbox.z || i < boundingbox.y || i >= boundingbox.w)
{
// mark as not useful
Camera3DPoints[idx].X = 0;
Camera3DPoints[idx].Y = 0;
Camera3DPoints[idx].Z = 0;
PfVec[idx] = defines.OUTSIDE_BB;
}
else // if it's inside the box
{
Color pixel = ColorTexture.GetPixel(j, i);
PfVec[idx] = GetPf(pixel);
// Draw the bounding box region
if (drawBox && OnBoxEdge(j, i))
{
//ColorTextureVisual.SetPixel( j, i, Color.blue );
ColorTextureVisual.SetPixel(j, i, Color.Lerp(pixel, Color.blue, 0.3f));
}
}
}
} //end for
}
/// <summary>
/// Use the current bounding box to assign foreground pixels
/// Everything else assign as background
/// This updates the vector Mask
/// </summary>
private void LabelMaskFromCurrentStateBoundingBox(HoloProxies.Engine.trackerState state)
{
// first get bounding box for foreground pixels
boundingbox = findBoundingBoxFromCurrentState(state);
// (x,y) ----- +
// | |
// | |
// + ----- (z,w)
// set the near background bounding box
UnityEngine.Vector4 boundingboxBG =
new UnityEngine.Vector4(boundingbox.x - defines.BB_MARGIN, //top, left
boundingbox.y - defines.BB_MARGIN,
boundingbox.z + defines.BB_MARGIN, //bottom, right
boundingbox.w + defines.BB_MARGIN);
for (int i = 0; i < Height; i++)
{
for (int j = 0; j < Width; j++)
{
int idx = i * Width + j;
// if it's not a zero depth pixel i.e. useless
//if (Mask[idx] != defines.HIST_USELESS_PIXEL)
//{
// if point is in the foreground bounding box
if (j > boundingbox.x && j < boundingbox.z && i > boundingbox.y && i < boundingbox.w)
{
Mask[idx] = defines.HIST_FG_PIXEL;
}
// else if it is in the near background
else if (j > boundingboxBG.x && j < boundingboxBG.z && i > boundingboxBG.y && i < boundingboxBG.w)
{
Mask[idx] = defines.HIST_BG_PIXEL;
}
else // else it is in the far background
{
Mask[idx] = defines.HIST_USELESS_PIXEL;
}
//}
}
} //end for
}
/// <summary>
/// Finds the state of the bounding box from current.
/// Project current pose points and create a bounding box that we will use
/// for the histogram
/// </summary>
/// <param name="state">State.</param>
private UnityEngine.Vector4 findBoundingBoxFromCurrentState(HoloProxies.Engine.trackerState state)
{
Vector3[] corners = new Vector3[8];
Vector3[] ipts = new Vector3[state.numPoses() * 8];
UnityEngine.Vector4 bb = new UnityEngine.Vector4(Width, Height, 0, 0);
for (int i = -1, idx = 0; i <= 1; i += 2)
{
for (int j = -1; j <= 1; j += 2)
{
for (int k = -1; k <= 1; k += 2, idx++)
{
corners[idx] = new Vector3(i * 0.075f, j * 0.075f, k * 0.075f);
}
}
}
for (int i = 0, idx = 0; i < state.numPoses(); i++)
{
for (int j = 0; j < 8; j++, idx++)
{
Matrix4x4 H = state.getPose(i).getH();
Vector3 temp = H.MultiplyPoint(corners[j]);
ipts[idx] = K * (H.MultiplyPoint(corners[j]));
ipts[idx].x /= ipts[idx].z; ipts[idx].y /= ipts[idx].z;
bb.x = ipts[idx].x < bb.x ? ipts[idx].x : bb.x;
bb.y = ipts[idx].y < bb.y ? ipts[idx].y : bb.y;
bb.z = ipts[idx].x > bb.z ? ipts[idx].x : bb.z;
bb.w = ipts[idx].y > bb.w ? ipts[idx].y : bb.w;
}
}
bb.x = bb.x < 0 ? 0 : bb.x;
bb.y = bb.y < 0 ? 0 : bb.y;
bb.z = bb.z > Width ? Width : bb.z;
bb.w = bb.w > Height ? Height : bb.w;
return(bb);
}
/// <summary>
/// Re-initializes the histogram based on current state.
/// Based on the pose, projects a foreground bounding box (everything
/// else is background) and stores it in Mask. Only the foreground pixels
/// are used to create the histogram.
/// </summary>
/// <param name="state">State.</param>
public void ReinitHistogramFromRendering(HoloProxies.Engine.trackerState state)
{
LabelMaskFromCurrentStateBoundingBox(state);
histogram.BuildHistogram(ColorTexture, Mask);
}
/// <summary>
/// Updates the frame by grabbing the rgb and depth from the Kinnect.
/// </summary>
/// <returns><c>true</c>, if frame was successfully updated, <c>false</c> otherwise.</returns>
/// <param name="state">State.</param>
public bool UpdateFrame(HoloProxies.Engine.trackerState state)
{
bool success = false;
if (_Reader != null)
{
var frame = _Reader.AcquireLatestFrame();
if (frame != null)
{
//Debug.Log( "in here1" );
var colorFrame = frame.ColorFrameReference.AcquireFrame();
if (colorFrame != null)
{
//Debug.Log( "in here2" );
var depthFrame = frame.DepthFrameReference.AcquireFrame();
if (depthFrame != null)
{
//Debug.Log( "in here3" );
// get color data + texture
colorFrame.CopyConvertedFrameDataToArray(ColorData_full, ColorImageFormat.Rgba);
// get depth data
depthFrame.CopyFrameDataToArray(DepthData_full);
// Create a depth texture
//int index = 0;
//foreach (var ir in DepthData_full)
//{
// byte intensity = (byte)(ir >> 8);
// DepthRaw[index++] = intensity;
// DepthRaw[index++] = intensity;
// DepthRaw[index++] = intensity;
// DepthRaw[index++] = 255; // Alpha
//}
//DepthTexture_full.LoadRawTextureData( DepthRaw );
//DepthTexture_full.Apply(); TODO fix this
// dispose frame
depthFrame.Dispose();
depthFrame = null;
// Map depth to RGBD
AlignRGBD();
// Unproject to 3D points in camera space (based on bounding box)
UnprojectFrom2Dto3D();
// Downsample all data frames if necessary
DownsampleAndFilterRGBDImage();
// Apply downsampled texture
ColorTexture.LoadRawTextureData(ColorData);
ColorTextureVisual.LoadRawTextureData(ColorData);
// Unproject points to 3D space
PreparePointCloud(state);
if (plotHistogram)
{
for (int i = 0; i < HistogramTexture.height; i++)
{
for (int j = 0; j < HistogramTexture.width; j++)
{
int idx = i * HistogramTexture.width + j;
HistogramTexture.SetPixel(j, i, new Color(histogram.posterior[idx * 256], 0, 0));
}
}
HistogramTexture.Apply();
}
success = true;
}
colorFrame.Dispose();
colorFrame = null;
}
frame = null;
}
}
return(success);
}