void Update() { if (foregroundCamera == null) { // by default use the main camera foregroundCamera = Camera.main; } if (kinectManager == null) { kinectManager = KinectManager.Instance; } if (kinectManager && kinectManager.IsInitialized() && foregroundCamera) { // get the background rectangle (use the portrait background, if available) Rect backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } // overlay the joints if (kinectManager.IsUserDetected(playerIndex)) { ulong userId = kinectManager.GetUserIdByIndex(playerIndex); OverlayJoint(userId, (int)KinectInterop.JointType.HandLeft, leftHandOverlay, backgroundRect); OverlayJoint(userId, (int)KinectInterop.JointType.HandRight, rightHandOverlay, backgroundRect); } } }
// scales the renderer's transform properly private void ScaleRendererTransform(Texture colorTex, int curScreenW, int curScreenH) { lastScreenW = curScreenW; lastScreenH = curScreenH; lastColorW = sensorData.colorImageWidth; lastColorH = sensorData.colorImageHeight; Vector3 localScale = transform.localScale; if (maximizeOnScreen && foregroundCamera) { float objectZ = transform.position.z; float screenW = foregroundCamera.pixelWidth; float screenH = foregroundCamera.pixelHeight; if (backgroundImage) { PortraitBackground portraitBack = backgroundImage.gameObject.GetComponent <PortraitBackground>(); if (portraitBack != null) { Rect backRect = portraitBack.GetBackgroundRect(); screenW = backRect.width; screenH = backRect.height; } } Vector3 vLeft = foregroundCamera.ScreenToWorldPoint(new Vector3(0f, screenH / 2f, objectZ)); Vector3 vRight = foregroundCamera.ScreenToWorldPoint(new Vector3(screenW, screenH / 2f, objectZ)); float distLeftRight = (vRight - vLeft).magnitude; Vector3 vBottom = foregroundCamera.ScreenToWorldPoint(new Vector3(screenW / 2f, 0f, objectZ)); Vector3 vTop = foregroundCamera.ScreenToWorldPoint(new Vector3(screenW / 2f, screenH, objectZ)); float distBottomTop = (vTop - vBottom).magnitude; localScale.x = distLeftRight / initialScale.x; localScale.y = distBottomTop / initialScale.y; //Debug.Log("ForegroundRenderer scale: " + localScale + ", screenW: " + screenW + ", screenH: " + screenH); } // scale according to color-tex resolution //localScale.y = localScale.x * colorTex.height / colorTex.width; // apply color image scale Vector3 colorImageScale = kinectManager.GetColorImageScale(backgroundRemovalManager.sensorIndex); if (colorImageScale.x < 0f) { localScale.x = -localScale.x; } if (colorImageScale.y < 0f) { localScale.y = -localScale.y; } transform.localScale = localScale; }
void Update() { if (!shaderRectInited) { PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.IsInitialized()) { shaderUvRect = portraitBack.GetShaderUvRect(); } shaderRectInited = true; } if (kinectManager && kinectManager.IsInitialized() && userBlendMat != null) { if (sensorData != null && sensorData.colorDepthBuffer != null && sensorData.colorImageTexture && lastColorDepthBufferTime != sensorData.lastColorDepthBufferTime) { lastColorDepthBufferTime = sensorData.lastColorDepthBufferTime; //Debug.Log("BodyBlender ColorDepthBufferTime: " + lastColorDepthBufferTime); userBlendMat.SetFloat("_ColorResX", (float)sensorData.colorImageWidth); userBlendMat.SetFloat("_ColorResY", (float)sensorData.colorImageHeight); userBlendMat.SetFloat("_ColorScaleX", (float)sensorData.colorImageScale.x); userBlendMat.SetFloat("_ColorOfsX", shaderUvRect.x); userBlendMat.SetFloat("_ColorMulX", shaderUvRect.width); userBlendMat.SetFloat("_ColorOfsY", shaderUvRect.y); userBlendMat.SetFloat("_ColorMulY", shaderUvRect.height); if (backgroundImage) { userBlendMat.SetTexture("_BackTex", backgroundImage.texture); } // color camera texture //Texture colorTex = backManager && sensorData.color2DepthTexture ? (Texture)sensorData.color2DepthTexture : sensorData.colorImageTexture; //userBlendMat.SetTexture("_ColorTex", colorTex); userBlendMat.SetTexture("_ColorTex", sensorData.colorImageTexture); // sensorData.colorDepthTexture userBlendMat.SetBuffer("_DepthMap", sensorData.colorDepthBuffer); } } }
// gets the joint position in space private Vector3 GetJointPosition(KinectManager manager, int joint) { Vector3 vPosJoint = Vector3.zero; if (manager.IsJointTracked(currentUserId, joint)) { if (backgroundPlane && planeRectSet) { // get the plane overlay position vPosJoint = manager.GetJointPosColorOverlay(currentUserId, joint, sensorIndex, planeRect); vPosJoint.z = backgroundPlane.position.z; } else if (foregroundCamera) { // get the background rectangle (use the portrait background, if available) Rect backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } // get the color overlay position vPosJoint = manager.GetJointPosColorOverlay(currentUserId, joint, sensorIndex, foregroundCamera, backgroundRect); } // else if (vPosJoint == Vector3.zero) { vPosJoint = manager.GetJointPosition(currentUserId, joint); } } return(vPosJoint); }
void Update() { if (kinectManager && kinectManager.IsInitialized() && foregroundCamera) { // get the background rectangle (use the portrait background, if available) Rect backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } // overlay the joint ulong userId = kinectManager.GetUserIdByIndex(playerIndex); int iJointIndex = (int)trackedJoint; if (kinectManager.IsJointTracked(userId, iJointIndex)) { Vector3 posJoint = kinectManager.GetJointPosColorOverlay(userId, iJointIndex, sensorIndex, foregroundCamera, backgroundRect); if (posJoint != Vector3.zero && overlayObject) { if (horizontalOffset != 0f) { // add the horizontal offset Vector3 dirHorizOfs = overlayObject.InverseTransformDirection(new Vector3(horizontalOffset, 0, 0)); posJoint += dirHorizOfs; } if (verticalOffset != 0f) { // add the vertical offset Vector3 dirVertOfs = overlayObject.InverseTransformDirection(new Vector3(0, verticalOffset, 0)); posJoint += dirVertOfs; } if (forwardOffset != 0f) { // add the forward offset Vector3 dirFwdOfs = overlayObject.InverseTransformDirection(new Vector3(0, 0, forwardOffset)); posJoint += dirFwdOfs; } overlayObject.position = posJoint; if (rotateObject) { Quaternion rotJoint = kinectManager.GetJointOrientation(userId, iJointIndex, !objMirrored); rotJoint = initialRotation * rotJoint; overlayObject.rotation = rotationSmoothFactor > 0f ? Quaternion.Slerp(overlayObject.rotation, rotJoint, rotationSmoothFactor * Time.deltaTime) : rotJoint; } } } else { // make the overlay object invisible if (overlayObject && overlayObject.position.z > 0f) { Vector3 posJoint = overlayObject.position; posJoint.z = -10f; overlayObject.position = posJoint; } } } }
//////////////////////////////////////////////////////////////////////// void Awake() { instance = this; }
void Update() { if (kinectManager && kinectManager.IsInitialized()) { if (foregroundCamera) { // get the background rectangle (use the portrait background, if available) backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } } // overlay all joints in the skeleton if (kinectManager.IsUserDetected(playerIndex)) { ulong userId = kinectManager.GetUserIdByIndex(playerIndex); int jointsCount = kinectManager.GetJointCount(); //Debug.Log("Displaying user " + playerIndex + ", ID: " + userId + // ", body: " + kinectManager.GetBodyIndexByUserId(userId) + ", pos: " + kinectManager.GetJointKinectPosition(userId, 0)); for (int i = 0; i < jointsCount; i++) { int joint = i; if (kinectManager.IsJointTracked(userId, joint)) { Vector3 posJoint = foregroundCamera ? kinectManager.GetJointPosColorOverlay(userId, joint, sensorIndex, foregroundCamera, backgroundRect) : sensorTransform?kinectManager.GetJointKinectPosition(userId, joint, true) : kinectManager.GetJointPosition(userId, joint); //Debug.Log("U " + userId + " " + (KinectInterop.JointType)joint + " - pos: " + posJoint); if (sensorTransform) { posJoint = sensorTransform.TransformPoint(posJoint); } if (joints != null) { // overlay the joint if (posJoint != Vector3.zero) { joints[i].SetActive(true); joints[i].transform.position = posJoint; Quaternion rotJoint = kinectManager.GetJointOrientation(userId, joint, false); rotJoint = initialRotation * rotJoint; joints[i].transform.rotation = rotJoint; } else { joints[i].SetActive(false); } } if (lines[i] == null && linePrefab != null) { lines[i] = Instantiate(linePrefab) as LineRenderer; lines[i].transform.parent = transform; lines[i].gameObject.SetActive(false); } if (lines[i] != null) { // overlay the line to the parent joint int jointParent = (int)kinectManager.GetParentJoint((KinectInterop.JointType)joint); Vector3 posParent = foregroundCamera ? kinectManager.GetJointPosColorOverlay(userId, jointParent, sensorIndex, foregroundCamera, backgroundRect) : sensorTransform?kinectManager.GetJointKinectPosition(userId, jointParent, true) : kinectManager.GetJointPosition(userId, jointParent); if (sensorTransform) { posParent = sensorTransform.TransformPoint(posParent); } if (posJoint != Vector3.zero && posParent != Vector3.zero) { lines[i].gameObject.SetActive(true); //lines[i].SetVertexCount(2); lines[i].SetPosition(0, posParent); lines[i].SetPosition(1, posJoint); } else { lines[i].gameObject.SetActive(false); } } } else { if (joints[i] != null) { joints[i].SetActive(false); } if (lines[i] != null) { lines[i].gameObject.SetActive(false); } } } } else { // disable the skeleton int jointsCount = kinectManager.GetJointCount(); for (int i = 0; i < jointsCount; i++) { if (joints[i] != null) { joints[i].SetActive(false); } if (lines[i] != null) { lines[i].gameObject.SetActive(false); } } } } }
// ----- end of public functions ----- void Update() { KinectManager kinectManager = KinectManager.Instance; if (kinectManager && kinectManager.IsInitialized()) { ////backgroundImage.renderer.material.mainTexture = kinectManager.GetUsersClrTex(); //if (backgroundImage && (backgroundImage.texture == null)) //{ // backgroundImage.texture = kinectManager.GetUsersClrTex(); //} // overlay the joint int iJointIndex = !isLeftHanded ? (int)KinectInterop.JointType.HandRight : (int)KinectInterop.JointType.HandLeft; if (kinectManager.IsUserDetected(playerIndex)) { ulong userId = kinectManager.GetUserIdByIndex(playerIndex); if (kinectManager.IsJointTracked(userId, iJointIndex)) { Vector3 posJointRaw = kinectManager.GetJointKinectPosition(userId, iJointIndex, false); //Vector3 posJoint = manager.GetJointPosColorOverlay(userId, iJointIndex, Camera.main, Camera.main.pixelRect); if (posJointRaw != Vector3.zero) { Vector2 posDepth = kinectManager.MapSpacePointToDepthCoords(sensorIndex, posJointRaw); ushort depthValue = kinectManager.GetDepthForPixel(sensorIndex, (int)posDepth.x, (int)posDepth.y); if (posDepth != Vector2.zero && depthValue > 0) { // depth pos to color pos Vector2 posColor = kinectManager.MapDepthPointToColorCoords(sensorIndex, posDepth, depthValue); if (posColor.x != 0f && !float.IsInfinity(posColor.x)) { KinectInterop.SensorData sensorData = kinectManager.GetSensorData(sensorIndex); // get the color image x-offset and width (use the portrait background, if available) float colorWidth = sensorData.colorImageWidth; float colorOfsX = 0f; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { colorWidth = portraitBack.GetColorScaledScreenWidth(); colorOfsX = (sensorData.colorImageWidth - colorWidth) / 2f; } float xScaled = (posColor.x - colorOfsX) / colorWidth; float yScaled = posColor.y / sensorData.colorImageHeight; float xScreen = (sensorData.colorImageScale.x > 0f ? xScaled : 1f - xScaled); float yScreen = (sensorData.colorImageScale.y > 0f ? yScaled : 1f - yScaled); cursorPos = Vector2.Lerp(cursorPos, new Vector2(xScreen, yScreen), smoothFactor * Time.deltaTime); //Debug.Log("CursorPos: " + new Vector2(xScreen, yScreen)); } } } } } } }
void Update() { if (kinectManager && kinectManager.IsInitialized()) { if (foregroundCamera) { // get the background rectangle (use the portrait background, if available) backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } } // overlay all joints in the skeleton if (kinectManager.IsSensorBodyDetected(sensorIndex, playerIndex)) { int jointsCount = kinectManager.GetJointCount(); if (joints == null && jointPrefab != null) { // array holding the skeleton joints joints = new GameObject[jointsCount]; for (int i = 0; i < joints.Length; i++) { joints[i] = Instantiate(jointPrefab) as GameObject; joints[i].transform.parent = transform; joints[i].name = ((KinectInterop.JointType)i).ToString(); joints[i].SetActive(false); Renderer renderer = joints[i].GetComponent <Renderer>(); if (renderer != null) { renderer.material.color = skeletonColor; } } } if (lines == null) { // array holding the skeleton lines lines = new LineRenderer[jointsCount]; } for (int i = 0; i < jointsCount; i++) { int joint = i; if (kinectManager.IsSensorJointTracked(sensorIndex, playerIndex, joint)) { Vector3 posJoint = GetJointPosition(joint); if (sensorTransform) { posJoint = sensorTransform.TransformPoint(posJoint); } if (joints != null) { // overlay the joint if (posJoint != Vector3.zero) { joints[i].SetActive(true); joints[i].transform.position = posJoint; Quaternion rotJoint = kinectManager.GetSensorJointOrientation(sensorIndex, playerIndex, joint, false); rotJoint = initialRotation * rotJoint; joints[i].transform.rotation = rotJoint; //if (i == (int)KinectInterop.JointType.WristLeft) //{ // Debug.Log(string.Format("SSO {0:F3} {1} user: {2}, state: {3}\npos: {4}, rot: {5}", Time.time, (KinectInterop.JointType)i, // playerIndex, kinectManager.GetSensorJointTrackingState(sensorIndex, playerIndex, joint), // kinectManager.GetSensorJointPosition(sensorIndex, playerIndex, joint), // kinectManager.GetSensorJointOrientation(sensorIndex, playerIndex, joint, false).eulerAngles)); //} } else { joints[i].SetActive(false); } } if (lines[i] == null && linePrefab != null) { lines[i] = Instantiate(linePrefab) as LineRenderer; lines[i].transform.parent = transform; lines[i].gameObject.SetActive(false); lines[i].startColor = skeletonColor; lines[i].endColor = skeletonColor; } if (lines[i] != null) { // overlay the line to the parent joint int jointParent = (int)kinectManager.GetParentJoint((KinectInterop.JointType)joint); Vector3 posParent = GetJointPosition(jointParent); if (sensorTransform) { posParent = sensorTransform.TransformPoint(posParent); } if (posJoint != Vector3.zero && posParent != Vector3.zero) { lines[i].gameObject.SetActive(true); //lines[i].SetVertexCount(2); lines[i].SetPosition(0, posParent); lines[i].SetPosition(1, posJoint); } else { lines[i].gameObject.SetActive(false); } } } else { if (joints[i] != null) { joints[i].SetActive(false); } if (lines[i] != null) { lines[i].gameObject.SetActive(false); } } } } else { // disable the skeleton int jointsCount = kinectManager.GetJointCount(); for (int i = 0; i < jointsCount; i++) { if (joints != null && joints[i] != null) { joints[i].SetActive(false); } if (lines != null && lines[i] != null) { lines[i].gameObject.SetActive(false); } } } } }