/// <summary>
/// Converts an ImageData type of the person mask to a color32 array.
/// This function does not call Texture2D.apply. which should be called to apply the changes to the texture
/// </summary>
/// <param name="maskImageData">Contains the mask image data</param>
/// <param name="maskColor">the color to set the mask to</param>
/// <param name="pixels">The color array to write the person mask on.
/// It must fit the image size, or else an error is returned</param>
/// <returns>ReturnCode indicating status of the run</returns>
public static ReturnCode ConvertPersonMaskToColorArray(MaskImageData maskImageData, Color32 maskColor, bool flipx, bool flipy, ref Color32[] pixels)
{
CommonDefines.ImageData imageData;
ReturnCode rc = maskImageData.GetData(out imageData);
if (rc.IsError())
{
return(rc);
}
CommonDefines.ImageFormat iFormat;
maskImageData.GetImageFormat(out iFormat);
rc = VerifyImageFormat(iFormat, pixels);
if (rc.IsError())
{
return(rc);
}
int xStart, xDir, yStart, yDir;
SetLoopVars(flipx, flipy, iFormat, out xStart, out xDir, out yStart, out yDir);
for (int y = 0; y != iFormat.m_height; ++y)
{
for (int x = 0; x != iFormat.m_width; ++x)
{
int pos = (yStart + yDir * y) * iFormat.m_widthStep + xStart + xDir * x;
int pixelPos = y * iFormat.m_widthStep + x;
pixels[pixelPos] = (imageData.m_dataArr[pos] > 0) ? maskColor : ZERO_COLOR;
}
}
return(ReturnCode.OK_Status);
}
// Update is called once per frame - call Framework's BeckonManager update method
void Update()
{
if (m_isInitilzed)
{
HandleInput();
if (!m_sequenceEnded)
{
ReturnCode rc = BeckonManager.BeckonInstance.UpdateStates(out m_hasNewImage);
if (rc.IsError())
{
if (rc.SDKReturnCode == Omek.OMKStatus.OMK_ERROR_SEQUENCE_EOF_REACHED || rc.SDKReturnCode == Omek.OMKStatus.OMK_ERROR_ASSERTION_FAILURE)
{
m_sequenceEnded = true;
m_hasNewImage = false;
Debug.LogWarning(new ReturnCode(Omek.OMKStatus.OMK_ERROR_SEQUENCE_EOF_REACHED).ToString());
}
else
{
Debug.LogError(rc.ToString());
if (rc.FrameworkReturnCode == ReturnCode.FrameworkReturnCodes.Sensor_Not_Initialized)
{
m_isInitilzed = false;
}
return;
}
}
}
}
}
// Update is called once per frame
// to achieve better performance the task is split to 2: updating the color array and apply the texture
// each of the 2 task is performed every other frame
void Update()
{
if (m_texture == null)
{
return;
}
if (Time.frameCount % 2 == 0 && !m_firstFrame) // should we apply the texture this frame or update the Color32 Array
{
m_texture.Apply();
return;
}
bool hasNewImage = false;
BeckonManager.BeckonInstance.HasNewImage(m_lastFrame, out m_lastFrame, out hasNewImage);
if (hasNewImage)
{
ReturnCode rc = UnitySensorImageRelated.UnityImageConverters.ConvertImageToColorArray(regularImageData, flipHorizontally, !flipVertically, ref m_pixels);
if (!rc.IsError())
{
m_texture.SetPixels32(m_pixels, 0);
if (m_firstFrame)
{
m_firstFrame = false;
m_texture.Apply();
}
}
}
}
// Update is called once per frame
public void Update()
{
if (!BeckonManager.BeckonInstance.IsInit())
{
return;
}
SetParameterFromUnity();
ReturnCode rc = m_transformer.UpdateState();
if (!rc.IsError())
{
Vector3 localPosition = transform.localPosition;
if (ControlXAxis)
{
localPosition.x = m_transformer.CurrentValue.x;
}
if (ControlYAxis)
{
localPosition.y = m_transformer.CurrentValue.y;
}
if (ControlZAxis)
{
localPosition.z = m_transformer.CurrentValue.z;
}
transform.localPosition = localPosition;
}
}
public bool ReinitSensor()
{
if (m_isInitilzed == false)
{
ReturnCode rc = BeckonManager.BeckonInstance.DestroySensor();
if (rc.IsError())
{
Debug.LogError(rc.ToString());
}
m_isInitilzed = CreateSensor();
}
return(m_isInitilzed);
}
/// <summary>
/// Center the world box so the CurrentValue will be at targetPosition
/// </summary>
/// <param name="targetPosition">the wanted position for CurrentValue, must be inside TargetBox</param>
/// <param name="smoothChange">should the change in position be smoothed or should it happen abruptly</param>
/// <returns>Return code that represent the status of the run</returns>
public bool RecenterOnTargetPosition(UnityEngine.Vector3 targetPosition, bool smoothChange = true)
{
ReturnCode rc = m_transformer.RecenterOnTargetPosition(UnityConverter.ToFramework(targetPosition), smoothChange);
if (!rc.IsError())
{
WorldBox.CenterOffset = UnityConverter.ToUnity(m_transformer.WorldBox.CenterOffset);
WorldBox.Dimensions = UnityConverter.ToUnity(m_transformer.WorldBox.Dimensions);
TargetBox.CenterOffset = UnityConverter.ToUnity(m_transformer.TargetBox.CenterOffset);
TargetBox.Dimensions = UnityConverter.ToUnity(m_transformer.TargetBox.Dimensions);
return(true);
}
else
{
Debug.LogError(rc);
return(false);
}
}
/// <summary>
/// Center the world box so the current position will be the center of the new box
/// </summary>
/// <param name="updateTargetBox">should the TargetBox be updated accordingly so the CurrentValue will stay in place</param>
/// <param name="smoothChange">should the change in position be smoothed or should it happen abruptly</param>
/// <returns>Return code that represent the status of the run</returns>
public bool RecenterOnWorldPosition(bool updateTargetBox, bool smoothChange)
{
ReturnCode rc = m_transformer.RecenterOnWorldPosition(updateTargetBox, smoothChange);
if (!rc.IsError())
{
WorldBox.CenterOffset = UnityConverter.ToUnity(m_transformer.WorldBox.CenterOffset);
WorldBox.Dimensions = UnityConverter.ToUnity(m_transformer.WorldBox.Dimensions);
TargetBox.CenterOffset = UnityConverter.ToUnity(m_transformer.TargetBox.CenterOffset);
TargetBox.Dimensions = UnityConverter.ToUnity(m_transformer.TargetBox.Dimensions);
return(true);
}
else
{
Debug.LogError(rc);
return(false);
}
}
// Update is called once per frame
public void Update()
{
SetParameterFromUnity();
ReturnCode rc = m_transformer.UpdateState();
if (!rc.IsError())
{
Vector3 localPosition = transform.localPosition;
if (ControlXAxis)
{
localPosition.x = m_transformer.CurrentValue.x;
CurrentValue.x = m_transformer.CurrentValue.x;
}
if (ControlYAxis)
{
localPosition.y = m_transformer.CurrentValue.y;
CurrentValue.y = m_transformer.CurrentValue.y;
}
transform.localPosition = localPosition;
}
}
// Update is called once per frame
void Update()
{
if (m_texture == null)
{
return;
}
bool hasNewImage = false;
BeckonManager.BeckonInstance.HasNewImage(m_lastFrame, out m_lastFrame, out hasNewImage);
if (hasNewImage)
{
ReturnCode rc = UnitySensorImageRelated.UnityImageConverters.ConvertImageToColorArray(regularImageData, flipHorizontally, !flipVertically, ref m_pixels);
if (!rc.IsError())
{
m_texture.SetPixels32(m_pixels, 0);
m_texture.Apply();
}
}
}
private bool CreateSensor()
{
//Debug.Log("m_currentConf " + m_currentConf.SequencePath);
ReturnCode rc = BeckonManager.BeckonInstance.CreateSensor(m_currentConf);
if (rc.IsError())
{
Debug.LogError("CreateSensor failed: " + rc);
return(false);
}
if (m_currentConf.UseRunSensor)
{
rc = new ReturnCode(BeckonManager.BeckonInstance.Analyzer.runTrackingThread());
if (rc.IsError())
{
Debug.LogError(rc.ToString());
return(false);
}
}
return(true);
}
/// <summary>
/// Converts an ImageData type of the depth or color image to a color32[] and masks it according to the entered person mask.
/// This function does not call Texture2D.apply. which should be called to apply the changes to the texture
/// </summary>
/// <param name="regImage">Contains the regular image data and format</param>
/// <param name="maskImage">Contains the person mask image data and format</param>
/// <param name="flipx">should the image be flipped horizontally</param>
/// <param name="flipy">should the image be flipped vertically</param>
/// <param name="depthColor">what is the base color of a depth image</param>
/// <param name="pixels">The Color32 Array to write to. mast be in the right size for the wanted operation.
/// <returns>ReturnCode indicating status of the run</returns>
public static ReturnCode ConvertImageToMaskedColorArray(RegularImageData regImage, MaskImageData maskImage, bool flipx, bool flipy, Color32 depthColor, ref Color32[] pixels)
{
// get the image format and data
CommonDefines.ImageFormat iFormat;
regImage.GetImageFormat(out iFormat);
CommonDefines.ImageData imageData;
ReturnCode rc = regImage.GetData(out imageData);
if (rc.IsError())
{
return(rc);
}
// get the mask format and data
CommonDefines.ImageFormat maskFormat;
maskImage.GetImageFormat(out maskFormat);
CommonDefines.ImageData maskedData;
rc = maskImage.GetData(out maskedData);
if (rc.IsError())
{
return(rc);
}
rc = VerifyImageFormat(iFormat, pixels);
if (rc.IsError())
{
return(rc);
}
int imagePos = 0;
int pixelPos = 0;
int bpp = iFormat.m_bpc;
int bppTimeChannels = iFormat.m_channels * bpp;
int imagePixelHeight = iFormat.m_height;
int imagePixelWidth = iFormat.m_width;
int formatWidthStep = iFormat.m_widthStep;
int maskHeightRatio = iFormat.m_height / maskFormat.m_height;
int maskWidthRatio = iFormat.m_width / maskFormat.m_width;
int maskImageWidth = maskFormat.m_width;
// parameters for normalizing
ushort minVal1 = 0, maxVal1 = 0;
ushort minVal2 = 0, maxVal2 = 0;
float range1 = 0, range2 = 0;
bool isGrayscale = (iFormat.m_channels == 1);
bool isDepth = (regImage.GetType() == typeof(DepthImageData));
minVal1 = (ushort)regImage.MinPixelValue;
maxVal1 = (ushort)regImage.MaxPixelValue;
range1 = maxVal1 - minVal1;
if (isDepth)
{
GetPixelRange((DepthImageData)regImage, maskImage, out minVal2, out maxVal2, out range2);
}
else // on regualr images we dont want to tint the color
{
depthColor = WHITE_COLOR;
}
// get the loop vlues - this handle fliping the image horizontally and verticlly
int xStart, xDir, yStart, yDir;
SetLoopVars(flipx, flipy, iFormat, out xStart, out xDir, out yStart, out yDir);
// iterate through the image
for (int yindex = 0; yindex != imagePixelHeight; ++yindex)
{
for (int xindex = 0; xindex != imagePixelWidth; ++xindex)
{
int y = yStart + yDir * yindex;
int x = xStart + xDir * xindex;
// for each pixel get the coresponding pixel in the player mask
int maskPos = (y / maskHeightRatio) * maskImageWidth + (x / maskWidthRatio);
byte maskVal = maskedData.m_dataArr[maskPos];
// get the pixel in the image
imagePos = y * formatWidthStep + x * bppTimeChannels;
pixelPos = yindex * imagePixelWidth + xindex;
if (maskVal > 0)
{
// get the image value and normalize if needed
if (isGrayscale)
{
// values are ushorts, get them from a byte[]
uint val1 = (uint)((imageData.m_dataArr[imagePos + 1] << 8) + imageData.m_dataArr[imagePos]);
// normalize according to the whole image
int val = (ushort)((val1 - minVal1) / (float)range1 * ushort.MaxValue);
float relativeVal = (float)val / ushort.MaxValue;
// we get here only in depth, we normalize according to the specific person and clamp values to [0.1,1]
if (range2 != 0)
{
relativeVal = (maxVal2 - val) / ((float)range2);
relativeVal = (relativeVal < 0.1f) ? 0.1f : relativeVal;
}
else // in normal grayscale image we clamp values to [0,1]
{
relativeVal = (relativeVal <= 0) ? 1f : relativeVal;
}
relativeVal = (relativeVal > 1f) ? 1f : relativeVal;
pixels[pixelPos].a = depthColor.a;
pixels[pixelPos].r = (byte)(depthColor.r * relativeVal);
pixels[pixelPos].g = (byte)(depthColor.g * relativeVal);
pixels[pixelPos].b = (byte)(depthColor.b * relativeVal);
}
else
{
pixels[pixelPos].a = 255;
pixels[pixelPos].r = imageData.m_dataArr[imagePos];
pixels[pixelPos].g = (imageData.m_dataArr[imagePos + 1]);
pixels[pixelPos].b = (imageData.m_dataArr[imagePos + 2]);
}
}
else
{
pixels[pixelPos] = ZERO_COLOR;
}
}
}
return(ReturnCode.OK_Status);
}
/// <summary>
/// Converts an SensorImageData type of the depth or color image to a color32 array.
/// This function does not call Texture2D.apply. which should be called to apply the changes to the texture
/// </summary>
/// <param name="sensorImageData">Contains the sensor image data</param>
/// /// <param name="flipx">should the image be flipped horizontally</param>
/// <param name="flipy">should the image be flipped vertically</param>
/// <param name="pixels">The Color32 Array to write to. mast be in the right size for the wanted operation.
/// <returns>ReturnCode indicating status of the run</returns>
public static ReturnCode ConvertImageToColorArray(RegularImageData sensorImageData, bool flipx, bool flipy, ref Color32[] pixels)
{
CommonDefines.ImageFormat iFormat;
sensorImageData.GetImageFormat(out iFormat);
CommonDefines.ImageData imageData;
ReturnCode rc = sensorImageData.GetData(out imageData);
if (rc.IsError())
{
return(rc);
}
rc = VerifyImageFormat(iFormat, pixels);
if (rc.IsError())
{
return(rc);
}
int imagePos = 0;
int pixelPos = 0;
int bpp = iFormat.m_bpc;
int bppTimeChannels = iFormat.m_channels * bpp;
int imagePixelHeight = iFormat.m_height;
int imagePixelWidth = iFormat.m_width;
int formatWidthStep = iFormat.m_widthStep;
// convert from 2-bpp to 1-bpp
ushort fromVal = 0, toVal = 0;
float range = 0;
bool isGrayscale = (iFormat.m_channels == 1);
bool isDepth = (sensorImageData.GetType() == typeof(DepthImageData));
if (isGrayscale)
{
fromVal = (ushort)sensorImageData.MinPixelValue;
toVal = (ushort)sensorImageData.MaxPixelValue;
range = toVal - fromVal;
}
// if we use depth image we want to reverse the grayscale - white for close pixels and black for far ones.
if (isDepth)
{
fromVal = toVal;
range *= -1;
}
int xStart, xDir, yStart, yDir;
SetLoopVars(flipx, flipy, iFormat, out xStart, out xDir, out yStart, out yDir);
for (int yindex = 0; yindex != imagePixelHeight; ++yindex)
{
for (int xindex = 0; xindex != imagePixelWidth; ++xindex)
{
int y = yStart + yDir * yindex;
int x = xStart + xDir * xindex;
imagePos = y * formatWidthStep + x * bppTimeChannels;
pixelPos = yindex * imagePixelWidth + xindex;
if (isGrayscale)
{
if ((imagePos + 1) <= imageData.m_dataArr.Length)
{
int intVal = (imageData.m_dataArr[imagePos + 1] << 8) + imageData.m_dataArr[imagePos];
// if isDepth values of 0 are actually no data, not very close to camera
if (intVal <= 0)
{
pixels[pixelPos].r = pixels[pixelPos].g = pixels[pixelPos].b = 0;
pixels[pixelPos].a = 255;
}
else
{
float val = (intVal - fromVal) / range;
val = val < 0f ? 0f : val;
val = val > 1f ? 1f : val;
pixels[pixelPos].a = 255;
pixels[pixelPos].r = pixels[pixelPos].g = pixels[pixelPos].b = (byte)(255 * val);
}
}
}
else
{
pixels[pixelPos].a = 255;
pixels[pixelPos].r = imageData.m_dataArr[imagePos];
pixels[pixelPos].g = (imageData.m_dataArr[imagePos + 1]);
pixels[pixelPos].b = (imageData.m_dataArr[imagePos + 2]);
}
}
}
return(ReturnCode.OK_Status);
}
