void Update()
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
Texture foregroundTex = backManager.GetForegroundTex();
if (backgroundImage && (backgroundImage.texture == null) && foregroundTex)
{
backgroundImage.texture = foregroundTex;
}
}
KinectManager manager = KinectManager.Instance;
if (manager && manager.IsInitialized())
{
if (currentCameraOffset != adjustedCameraOffset)
{
// update the camera automatically, according to the current sensor height and angle
KinectInterop.SensorData sensorData = manager.GetSensorData();
if (foregroundCamera != null && sensorData != null)
{
Vector3 fgCameraPos = initialCameraPos;
fgCameraPos.x += sensorData.faceOverlayOffset + adjustedCameraOffset;
foregroundCamera.transform.position = fgCameraPos;
currentCameraOffset = adjustedCameraOffset;
}
}
}
}
void Update()
{
if (rawImage && rawImage.texture == null)
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && backManager && backManager.enabled /**&& backManager.IsBackgroundRemovalInitialized()*/)
{
Debug.Log("foregraoundrawimage1");
rawImage.texture = backManager.GetForegroundTex(); // user's foreground texture
rawImage.rectTransform.localScale = kinectManager.GetColorImageScale();
rawImage.color = Color.white;
}
else if (kinectManager /**&& kinectManager.IsInitialized()*/)
{
Debug.Log("foregroundrawimage2");
SimpleBackgroundRemoval simpleBR = GameObject.FindObjectOfType <SimpleBackgroundRemoval>();
bool isSimpleBR = simpleBR && simpleBR.enabled;
rawImage.texture = kinectManager.GetUsersClrTex(); // color camera texture
rawImage.rectTransform.localScale = kinectManager.GetColorImageScale();
rawImage.color = !isSimpleBR ? Color.white : Color.clear;
}
}
// else if(rawImage && rawImage.texture != null)
// {
// KinectManager kinectManager = KinectManager.Instance;
// if(kinectManager == null)
// {
// rawImage.texture = null;
// rawImage.color = Color.clear;
// }
// }
}
/// <summary>
/// 获取实时图像
/// </summary>
/// <param name="orbbecImageType"></param>
/// <returns></returns>
public Texture GetOrbbecImage()
{
if (backManager == null)
{
backManager = KinectManager.Instance.gameObject.AddComponent <BackgroundRemovalManager>();
}
return(backManager.GetForegroundTex());
}
void Update()
{
if (manager == null || !manager.IsInitialized())
{
return;
}
// get user texture
if (GetComponent <Renderer>().material.mainTexture == null)
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
GetComponent <Renderer>().material.mainTexture = backManager ? (Texture)sensorData.depth2ColorTexture : (Texture)manager.GetUsersLblTex();
}
// get kinect-to-world matrix
kinectToWorld = manager.GetKinectToWorldMatrix();
if (playerIndex >= 0)
{
long lastUserId = userId;
userId = manager.GetUserIdByIndex(playerIndex);
userBodyIndex = (byte)manager.GetBodyIndexByUserId(userId);
if (userBodyIndex == 255)
{
userBodyIndex = 222;
}
// get user mesh position
userMeshPos = manager.GetJointKinectPosition(userId, (int)KinectInterop.JointType.SpineBase);
if (!mirroredMovement)
{
userMeshPos.x = -userMeshPos.x;
}
userMeshPos = kinectToWorld.MultiplyPoint3x4(userMeshPos); // convert to world coords
// set transform position
Vector3 newUserPos = manager.GetJointPosition(userId, (int)KinectInterop.JointType.SpineBase) + originPosition;
if (invertedZMovement)
{
newUserPos.z = -newUserPos.z;
}
transform.position = lastUserId != 0 && smoothFactor != 0f ? Vector3.Lerp(transform.position, newUserPos, smoothFactor * Time.deltaTime) : newUserPos;
}
else
{
userId = 0;
userBodyIndex = 255;
userMeshPos = Vector3.zero;
}
// update the mesh
UpdateMesh();
}
void OnDestroy()
{
if (isBrInited && sensorData != null && sensorData.sensorInterface != null)
{
// finish background removal
sensorData.sensorInterface.FinishBackgroundRemoval(sensorData);
}
isBrInited = false;
instance = null;
}
void Update()
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
if (thisRenderer && backManager && backManager.IsBackgroundRemovalInitialized())
{
if (thisRenderer.sharedMaterial.mainTexture == null)
{
thisRenderer.sharedMaterial.mainTexture = backManager.GetForegroundTex();
}
}
}
void Update()
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
if (img && img.texture == null)
{
//img.texture = backManager.GetForegroundTex();
img.texture = backManager.GetAlphaBodyTex();
}
}
}
void Start()
{
kinectManager = KinectManager.Instance;
backManager = BackgroundRemovalManager.Instance;
if (kinectManager && kinectManager.IsInitialized() &&
(kinectManager.GetSensorData().sensorIntPlatform == KinectInterop.DepthSensorPlatform.KinectSDKv2
/**|| kinectManager.GetSensorData().sensorIntPlatform == KinectInterop.DepthSensorPlatform.OrbbecAstra*/))
{
Shader userBlendShader = Shader.Find("Custom/UserBlendShader");
KinectInterop.SensorData sensorData = kinectManager.GetSensorData();
if (userBlendShader != null && sensorData != null)
{
userBlendMat = new Material(userBlendShader);
userBlendMat.SetFloat("_ColorResX", (float)sensorData.colorImageWidth);
userBlendMat.SetFloat("_ColorResY", (float)sensorData.colorImageHeight);
userBlendMat.SetFloat("_DepthResX", (float)sensorData.depthImageWidth);
userBlendMat.SetFloat("_DepthResY", (float)sensorData.depthImageHeight);
//depthFactor = 1f + Mathf.Sin (Mathf.Abs (kinectManager.sensorAngle) * Mathf.Deg2Rad);
//userBlendMat.SetFloat("_DepthFactor", depthFactor);
color2DepthCoords = new Vector2[sensorData.colorImageWidth * sensorData.colorImageHeight];
color2DepthBuffer = new ComputeBuffer(sensorData.colorImageWidth * sensorData.colorImageHeight, sizeof(float) * 2);
userBlendMat.SetBuffer("_DepthCoords", color2DepthBuffer);
depthImageBufferData = new float[sensorData.depthImage.Length];
depthImageBuffer = new ComputeBuffer(sensorData.depthImage.Length, sizeof(float));
userBlendMat.SetBuffer("_DepthBuffer", depthImageBuffer);
if (backgroundImage)
{
userBlendMat.SetTexture("_BackTex", backgroundImage.texture);
}
// color camera texture
//userBlendMat.SetTexture("_ColorTex", sensorData.colorImageTexture);
// Texture colorTex = backManager && sensorData.color2DepthTexture ? (Texture)sensorData.color2DepthTexture : (Texture)sensorData.colorImageTexture;
// userBlendMat.SetTexture("_ColorTex", colorTex);
}
}
else
{
// disable the component
gameObject.GetComponent <UserBodyBlender>().enabled = false;
}
}
void Update()
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
GUITexture guiTexture = GetComponent <GUITexture>();
if (guiTexture && guiTexture.texture == null)
{
guiTexture.texture = backManager.GetForegroundTex();
}
}
}
void Update()
{
transform.localPosition = Player.JointOffset;
transform.localScale = new Vector3(Player.AspectScale.x * Player.JointScale, -Player.AspectScale.y * Player.JointScale, Player.AspectScale.z * Player.JointScale);
// Render the kinect color image onto a quad
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
var tex = backManager.GetForegroundTex();
MeshRenderer.material.SetTexture("_MainTex", tex);
}
}
void Start()
{
kinectManager = KinectManager.Instance;
initialScale = transform.localScale;
if (backgroundRemovalManager == null)
{
backgroundRemovalManager = FindObjectOfType <BackgroundRemovalManager>();
}
Renderer meshRenderer = GetComponent <Renderer>();
if (meshRenderer)
{
Shader blendShader = Shader.Find("Kinect/ForegroundBlendShader");
if (blendShader != null)
{
matRenderer = new Material(blendShader);
meshRenderer.material = matRenderer;
}
}
if (kinectManager && kinectManager.IsInitialized() && backgroundRemovalManager && backgroundRemovalManager.enabled)
{
// get sensor data
sensorData = kinectManager.GetSensorData(backgroundRemovalManager.sensorIndex);
sensorInt = sensorData != null ? (DepthSensorBase)sensorData.sensorInterface : null;
}
if (foregroundCamera == null)
{
foregroundCamera = Camera.main;
}
// find scene lights
Light[] sceneLights = GameObject.FindObjectsOfType <Light>();
lighting.SetLightsAndBounds(sceneLights, transform.position, new Vector3(20f, 20f, 20f));
//Debug.Log("sceneLights: " + sceneLights.Length);
//for(int i = 0; i < sceneLights.Length; i++)
//{
// Debug.Log(i.ToString() + " - " + sceneLights[i].name + " - " + sceneLights[i].type);
//}
}
void Update()
{
if (foregroundBlendMat && backgroundTexture &&
kinectManager && kinectManager.IsInitialized())
{
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
}
Texture alphaBodyTex = backManager ? backManager.GetAlphaBodyTex() : null;
KinectInterop.SensorData sensorData = kinectManager.GetSensorData();
if (backManager && backManager.IsBackgroundRemovalInitialized() &&
alphaBodyTex && backgroundTexture && lastDepthFrameTime != sensorData.lastDepthFrameTime)
{
lastDepthFrameTime = sensorData.lastDepthFrameTime;
foregroundBlendMat.SetTexture("_BodyTex", alphaBodyTex);
}
}
}
void Update()
{
if (thisRenderer && thisRenderer.sharedMaterial.mainTexture == null)
{
KinectManager kinectManager = KinectManager.Instance;
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
if (kinectManager && backManager && backManager.enabled /**&& backManager.IsBackgroundRemovalInitialized()*/)
{
thisRenderer.sharedMaterial.mainTexture = backManager.GetForegroundTex();
}
}
// else if(thisRenderer && thisRenderer.sharedMaterial.mainTexture != null)
// {
// KinectManager kinectManager = KinectManager.Instance;
// if(kinectManager == null)
// {
// thisRenderer.sharedMaterial.mainTexture = null;
// }
// }
}
void Update()
{
if (rawImage && rawImage.texture == null)
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && backManager && backManager.enabled /**&& backManager.IsBackgroundRemovalInitialized()*/)
{
Debug.Log("VideoRawImage1");
rawImage.texture = kinectManager.GetUsersClrTex(); // color camera texture
rawImage.rectTransform.localScale = kinectManager.GetColorImageScale() * -1;
//rawImage.uvRect = new Rect(1, 0, -1, 1);
rawImage.color = Color.white;
}
else if (kinectManager /**&& kinectManager.IsInitialized()*/)
{
Debug.Log("videorawImage2");
SimpleBackgroundRemoval simpleBR = GameObject.FindObjectOfType <SimpleBackgroundRemoval>();
bool isSimpleBR = simpleBR && simpleBR.enabled;
Debug.Log("manager exists, doing something here");
rawImage.texture = kinectManager.GetUsersClrTex(); // color camera texture
rawImage.rectTransform.localScale = kinectManager.GetColorImageScale();
rawImage.color = !isSimpleBR ? Color.white : Color.clear;
}
}
// else if(rawImage && rawImage.texture != null)
// {
// KinectManager kinectManager = KinectManager.Instance;
// if(kinectManager == null)
// {
// rawImage.texture = null;
// rawImage.color = Color.clear;
// }
// }
}
void OnDestroy()
{
if(isBrInited && sensorData != null && sensorData.sensorInterface != null)
{
// finish background removal
sensorData.sensorInterface.FinishBackgroundRemoval(sensorData);
}
isBrInited = false;
instance = null;
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId == 0)
{
if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
{
targetObject.material.mainTexture = null;
}
return;
}
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
if (backManager)
{
// re-initialize the texture
colorTex = null;
}
}
if (!bSkipUpdatingForegroundTex && backManager && backManager.IsBackgroundRemovalInitialized())
{
// use foreground image
Texture bmForeTex = backManager.GetForegroundTex();
if (bmForeTex is RenderTexture)
{
foregroundTex = (RenderTexture)bmForeTex;
bSkipUpdatingForegroundTex = (foregroundTex != null);
}
else
{
if (foregroundTex == null)
{
foregroundTex = new RenderTexture(bmForeTex.width, bmForeTex.height, 0);
}
Graphics.Blit(bmForeTex, foregroundTex);
bSkipUpdatingForegroundTex = false;
}
// if (!colorTex)
// {
// colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
// }
//
// try
// {
// foregroundTex = (RenderTexture)backManager.GetForegroundTex ();
// //KinectInterop.RenderTex2Tex2D (foregroundTex, ref colorTex);
// }
// catch (System.Exception)
// {
// colorTex = (Texture2D)backManager.GetForegroundTex ();
// }
}
else
{
// use color camera image
if (!colorTex)
{
colorTex = kinectManager.GetUsersClrTex2D();
}
}
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (faceRect.width > 0 && faceRect.height > 0)
{
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if (foregroundTex)
{
if ((faceX + faceW) > foregroundTex.width)
{
faceW = foregroundTex.width - faceX;
}
if ((faceY + faceH) > foregroundTex.height)
{
faceH = foregroundTex.height - faceY;
}
}
else if (colorTex)
{
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
if (foregroundTex)
{
KinectInterop.RenderTex2Tex2D(foregroundTex, faceX, foregroundTex.height - faceY - faceH, faceW, faceH, ref faceTex);
}
else if (colorTex)
{
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
}
if (targetObject && !targetObject.gameObject.activeSelf)
{
targetObject.gameObject.SetActive(true);
}
if (targetObject && targetObject.material)
{
targetObject.material.mainTexture = faceTex;
}
// don't rotate the transform - mesh follows the head rotation
if (targetObject.transform.rotation != Quaternion.identity)
{
targetObject.transform.rotation = Quaternion.identity;
}
}
else
{
if (targetObject && targetObject.gameObject.activeSelf)
{
targetObject.gameObject.SetActive(false);
}
if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
{
targetObject.material.mainTexture = null;
}
}
}
// Start is called before the first frame update
void Start()
{
_KinectManager = KinectManager.Instance;
_BackgroundRemover = BackgroundRemovalManager.Instance;
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
//unsafe for editor but can optimise if commented
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);
if (userId == 0)
{
return;
}
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
if (backManager)
{
// re-initialize the texture
colorTex = null;
}
}
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
// use foreground image
if (!colorTex)
{
colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
}
try
{
RenderTexture foregroundTex = (RenderTexture)backManager.GetForegroundTex();
KinectInterop.RenderTex2Tex2D(foregroundTex, ref colorTex);
}
catch (System.Exception)
{
colorTex = (Texture2D)backManager.GetForegroundTex();
}
}
else
{
// use color camera image
colorTex = kinectManager.GetUsersClrTex();
}
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (!colorTex || faceRect.width <= 0 || faceRect.height <= 0)
{
return;
}
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
if (targetRenderer && targetRenderer.material)
{
targetRenderer.material.mainTexture = faceTex;
}
}
//----------------------------------- end of public functions --------------------------------------//
void Awake()
{
instance = this;
}
// opens the default sensor and needed readers
public static SensorData OpenDefaultSensor(List<DepthSensorInterface> listInterfaces, FrameSource dwFlags, float sensorAngle, bool bUseMultiSource,
KinectManager.UserMapType userMapType, BackgroundRemovalManager brManager)
{
SensorData sensorData = null;
if(listInterfaces == null)
return sensorData;
foreach(DepthSensorInterface sensorInt in listInterfaces)
{
try
{
if(sensorData == null)
{
sensorData = sensorInt.OpenDefaultSensor(dwFlags, sensorAngle, bUseMultiSource);
if(sensorData != null)
{
sensorData.sensorInterface = sensorInt;
sensorData.sensorIntPlatform = sensorInt.GetSensorPlatform();
Debug.Log("Interface used: " + sensorInt.GetType().Name);
Debug.Log("Shader level: " + SystemInfo.graphicsShaderLevel);
if(sensorData.bodyIndexImage != null && IsDirectX11Available())
{
Shader bodyIndexShader = Shader.Find("Kinect/BodyShader");
if(bodyIndexShader != null)
{
sensorData.bodyIndexTexture = new RenderTexture(sensorData.depthImageWidth, sensorData.depthImageHeight, 0);
sensorData.bodyIndexTexture.wrapMode = TextureWrapMode.Clamp;
sensorData.bodyIndexTexture.filterMode = FilterMode.Point;
//Debug.Log(sensorData.bodyIndexTexture.format);
sensorData.bodyIndexMaterial = new Material(bodyIndexShader);
sensorData.bodyIndexMaterial.SetFloat("_TexResX", (float)sensorData.depthImageWidth);
sensorData.bodyIndexMaterial.SetFloat("_TexResY", (float)sensorData.depthImageHeight);
sensorData.bodyIndexBuffer = new ComputeBuffer(sensorData.bodyIndexImage.Length, sizeof(float));
sensorData.bodyIndexMaterial.SetBuffer("_BodyIndexBuffer", sensorData.bodyIndexBuffer);
}
}
if(sensorData.depthImage != null && IsDirectX11Available() &&
userMapType == KinectManager.UserMapType.UserTexture)
{
Shader depthImageShader = Shader.Find("Kinect/DepthShader");
if(depthImageShader != null)
{
sensorData.depthImageTexture = new RenderTexture(sensorData.depthImageWidth, sensorData.depthImageHeight, 0);
sensorData.depthImageTexture.wrapMode = TextureWrapMode.Clamp;
sensorData.depthImageTexture.filterMode = FilterMode.Point;
sensorData.depthImageMaterial = new Material(depthImageShader);
sensorData.depthImageMaterial.SetTexture("_MainTex", sensorData.bodyIndexTexture);
sensorData.depthImageMaterial.SetFloat("_TexResX", (float)sensorData.depthImageWidth);
sensorData.depthImageMaterial.SetFloat("_TexResY", (float)sensorData.depthImageHeight);
sensorData.depthImageBuffer = new ComputeBuffer(sensorData.depthImage.Length, sizeof(float));
sensorData.depthImageMaterial.SetBuffer("_DepthBuffer", sensorData.depthImageBuffer);
sensorData.depthHistBuffer = new ComputeBuffer(5001, sizeof(float));
sensorData.depthImageMaterial.SetBuffer("_HistBuffer", sensorData.depthHistBuffer);
// use body index buffer to overcome the linear color correction
sensorData.depthImageMaterial.SetBuffer("_BodyIndexBuffer", sensorData.bodyIndexBuffer);
}
}
if(sensorData.colorImage != null)
{
sensorData.colorImageTexture = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false);
}
// check if background removal requires cut-out image
bool bBrRequiresCutOut = brManager && (!brManager.colorCameraResolution || !sensorInt.IsBRHiResSupported());
if(sensorData.bodyIndexImage != null && sensorData.colorImage != null && IsDirectX11Available() &&
(userMapType == KinectManager.UserMapType.CutOutTexture || bBrRequiresCutOut))
{
Shader depth2ColorShader = Shader.Find("Kinect/Depth2ColorShader");
if(depth2ColorShader)
{
sensorData.depth2ColorTexture = new RenderTexture(sensorData.depthImageWidth, sensorData.depthImageHeight, 0);
sensorData.depth2ColorTexture.wrapMode = TextureWrapMode.Clamp;
//sensorData.depth2ColorTexture.filterMode = FilterMode.Point;
sensorData.depth2ColorMaterial = new Material(depth2ColorShader);
sensorData.depth2ColorMaterial.SetFloat("_ColorResX", (float)sensorData.colorImageWidth);
sensorData.depth2ColorMaterial.SetFloat("_ColorResY", (float)sensorData.colorImageHeight);
sensorData.depth2ColorMaterial.SetFloat("_DepthResX", (float)sensorData.depthImageWidth);
sensorData.depth2ColorMaterial.SetFloat("_DepthResY", (float)sensorData.depthImageHeight);
sensorData.depth2ColorBuffer = new ComputeBuffer(sensorData.depthImage.Length, sizeof(float) * 2);
sensorData.depth2ColorMaterial.SetBuffer("_ColorCoords", sensorData.depth2ColorBuffer);
sensorData.depth2ColorCoords = new Vector2[sensorData.depthImage.Length];
}
}
// if(sensorData.bodyIndexImage != null && sensorData.colorImage != null && IsDirectX11Available() &&
// (userMapType == KinectManager.UserMapType.CutOutTexture || bBrRequiresCutOut))
// {
// sensorData.depth2ColorCoords = new Vector2[sensorData.depthImage.Length];
// }
}
}
else
{
sensorInt.FreeSensorInterface(false);
}
}
catch (Exception ex)
{
Debug.LogError("Initialization of the sensor failed.");
Debug.LogError(ex.ToString());
try
{
sensorInt.FreeSensorInterface(false);
}
catch (Exception)
{
// do nothing
}
}
}
return sensorData;
}
void Update()
{
if (manager == null || !manager.IsInitialized())
{
return;
}
// get user texture
Renderer renderer = GetComponent <Renderer>();
if (renderer && renderer.material && renderer.material.mainTexture == null)
{
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
renderer.material.mainTexture = backManager && backManager.IsBackgroundRemovalInitialized() ?
(Texture)sensorData.color2DepthTexture : (Texture)manager.GetUsersLblTex();
}
// get kinect-to-world matrix
kinectToWorld = manager.GetKinectToWorldMatrix();
if (playerIndex >= 0)
{
long lastUserId = userId;
userId = manager.GetUserIdByIndex(playerIndex);
userBodyIndex = (byte)manager.GetBodyIndexByUserId(userId);
if (userBodyIndex == 255)
{
userBodyIndex = 222;
}
// check for color overlay
if (foregroundCamera)
{
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if (portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
// get user position
userMeshPos = manager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.SpineBase, foregroundCamera, backgroundRect);
}
else
{
// get user position
userMeshPos = manager.GetJointKinectPosition(userId, (int)KinectInterop.JointType.SpineBase);
}
if (!mirroredMovement)
{
//userMeshPos.x = -userMeshPos.x;
userMeshPos.x = 0f;
}
if (foregroundCamera == null)
{
// convert kinect pos to world coords, when there is no color overlay
userMeshPos = kinectToWorld.MultiplyPoint3x4(userMeshPos);
}
// set transform position
Vector3 newUserPos = userMeshPos + originPosition; // manager.GetJointPosition(userId, (int)KinectInterop.JointType.SpineBase) + originPosition;
if (invertedZMovement)
{
newUserPos.z = -newUserPos.z;
}
transform.position = lastUserId != 0 && smoothFactor != 0f ? Vector3.Lerp(transform.position, newUserPos, smoothFactor * Time.deltaTime) : newUserPos;
}
else
{
userId = 0;
userBodyIndex = 255;
userMeshPos = Vector3.zero;
}
// update the mesh
UpdateMesh();
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Background removal cannot be started, because KinectManager is missing or not initialized.");
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
bool bSuccess = sensorData.sensorInterface.IsBackgroundRemovalAvailable(ref bNeedRestart);
if (bSuccess)
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "BR");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Background removal is not supported!");
}
// Initialize the background removal
bSuccess = sensorData.sensorInterface.InitBackgroundRemoval(sensorData, colorCameraResolution);
if (!bSuccess)
{
throw new Exception("Background removal could not be initialized.");
}
// create the foreground image and alpha-image
int imageLength = sensorData.sensorInterface.GetForegroundFrameLength(sensorData, colorCameraResolution);
foregroundImage = new byte[imageLength];
// get the needed rectangle
Rect neededFgRect = sensorData.sensorInterface.GetForegroundFrameRect(sensorData, colorCameraResolution);
// create the foreground texture
foregroundTex = new Texture2D((int)neededFgRect.width, (int)neededFgRect.height, TextureFormat.RGBA32, false);
// calculate the foreground rectangle
if (foregroundCamera != null)
{
Rect cameraRect = foregroundCamera.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if (rectWidth > rectHeight)
{
rectWidth = Mathf.Round(rectHeight * neededFgRect.width / neededFgRect.height);
}
else
{
rectHeight = Mathf.Round(rectWidth * neededFgRect.height / neededFgRect.width);
}
foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight);
}
instance = this;
isBrInited = true;
//DontDestroyOnLoad(gameObject);
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and BR-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if(kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if(sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Background removal cannot be started, because KinectManager is missing or not initialized.");
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
bool bSuccess = sensorData.sensorInterface.IsBackgroundRemovalAvailable(ref bNeedRestart);
if(bSuccess)
{
if(bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "BR");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Background removal is not supported!");
}
// Initialize the background removal
bSuccess = sensorData.sensorInterface.InitBackgroundRemoval(sensorData, colorCameraResolution);
if (!bSuccess)
{
throw new Exception("Background removal could not be initialized.");
}
// create the foreground image and alpha-image
int imageLength = sensorData.sensorInterface.GetForegroundFrameLength(sensorData, colorCameraResolution);
foregroundImage = new byte[imageLength];
// get the needed rectangle
Rect neededFgRect = sensorData.sensorInterface.GetForegroundFrameRect(sensorData, colorCameraResolution);
// create the foreground texture
foregroundTex = new Texture2D((int)neededFgRect.width, (int)neededFgRect.height, TextureFormat.RGBA32, false);
// calculate the foreground rectangle
if(foregroundCamera != null)
{
Rect cameraRect = foregroundCamera.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if(rectWidth > rectHeight)
rectWidth = Mathf.Round(rectHeight * neededFgRect.width / neededFgRect.height);
else
rectHeight = Mathf.Round(rectWidth * neededFgRect.height / neededFgRect.width);
foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight);
}
instance = this;
isBrInited = true;
//DontDestroyOnLoad(gameObject);
}
catch(DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and BR-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
private void UpdateMesh()
{
if (sensorData.depthImage != null && sensorData.bodyIndexImage != null &&
sensorData.depth2SpaceCoords != null && lastSpaceCoordsTime != sensorData.lastDepth2SpaceCoordsTime)
{
int vCount = 0, tCount = 0;
EstimateUserVertices(out vCount, out tCount);
vertices = new Vector3[vCount];
uvs = new Vector2[vCount];
triangles = new int[6 * tCount];
BackgroundRemovalManager backManager = BackgroundRemovalManager.Instance;
bool isBackManagerOn = backManager && backManager.IsBackgroundRemovalInitialized();
int index = 0, vIndex = 0, tIndex = 0, xyIndex = 0;
for (int y = 0; y < depthHeight; y += sampleSize)
{
int xyStartIndex = xyIndex;
for (int x = 0; x < depthWidth; x += sampleSize)
{
//Vector3 vSpacePos = spaceCoords[xyIndex];
Vector3 vSpacePos = sensorData.depth2SpaceCoords[xyIndex];
if (vertexType[index] != 0 &&
!float.IsInfinity(vSpacePos.x) && !float.IsInfinity(vSpacePos.y) && !float.IsInfinity(vSpacePos.z))
{
// check for color overlay
if (foregroundCamera)
{
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if (portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
Vector2 vColorPos = sensorData.depth2ColorCoords[xyIndex];
ushort depthValue = sensorData.depthImage[xyIndex];
if (!float.IsInfinity(vColorPos.x) && !float.IsInfinity(vColorPos.y) && depthValue > 0)
{
float xScaled = (float)vColorPos.x * backgroundRect.width / sensorData.colorImageWidth;
float yScaled = (float)vColorPos.y * backgroundRect.height / sensorData.colorImageHeight;
float xScreen = backgroundRect.x + xScaled;
float yScreen = backgroundRect.y + backgroundRect.height - yScaled;
float zDistance = (float)depthValue / 1000f;
vSpacePos = foregroundCamera.ScreenToWorldPoint(new Vector3(xScreen, yScreen, zDistance));
}
}
if (!mirroredMovement)
{
vSpacePos.x = -vSpacePos.x;
}
if (foregroundCamera == null)
{
// convert space to world coords, when there is no color overlay
vSpacePos = kinectToWorld.MultiplyPoint3x4(vSpacePos);
}
float u = (float)(isBackManagerOn && sensorData.colorImageScale.x < 0f ? depthWidth - x - 1 : x);
float v = (float)(isBackManagerOn && sensorData.colorImageScale.y > 0f ? depthHeight - y - 1 : y);
vertices[vIndex] = vSpacePos - userMeshPos;
uvs[vIndex] = new Vector2(u / depthWidth, v / depthHeight);
vIndex++;
if (vertexType[index] == 3)
{
if (mirroredMovement)
{
triangles[tIndex++] = vertexIndex[index]; // top left
triangles[tIndex++] = vertexIndex[index + 1]; // top right
triangles[tIndex++] = vertexIndex[index + sampledWidth]; // bottom left
triangles[tIndex++] = vertexIndex[index + sampledWidth]; // bottom left
triangles[tIndex++] = vertexIndex[index + 1]; // top right
triangles[tIndex++] = vertexIndex[index + sampledWidth + 1]; // bottom right
}
else
{
triangles[tIndex++] = vertexIndex[index + 1]; // top left
triangles[tIndex++] = vertexIndex[index]; // top right
triangles[tIndex++] = vertexIndex[index + sampledWidth + 1]; // bottom left
triangles[tIndex++] = vertexIndex[index + sampledWidth + 1]; // bottom left
triangles[tIndex++] = vertexIndex[index]; // top right
triangles[tIndex++] = vertexIndex[index + sampledWidth]; // bottom right
}
}
}
index++;
xyIndex += sampleSize;
}
xyIndex = xyStartIndex + sampleSize * depthWidth;
}
// buffer is released
lastSpaceCoordsTime = sensorData.lastDepth2SpaceCoordsTime;
// lock(sensorData.spaceCoordsBufferLock)
// {
// sensorData.spaceCoordsBufferReady = false;
// }
mesh.Clear();
mesh.vertices = vertices;
mesh.uv = uvs;
//mesh.normals = normals;
mesh.triangles = triangles;
mesh.RecalculateNormals();
mesh.RecalculateBounds();
if (updateMeshCollider)
{
MeshCollider meshCollider = GetComponent <MeshCollider>();
if (meshCollider)
{
meshCollider.sharedMesh = null;
meshCollider.sharedMesh = mesh;
}
}
}
}
