void Update() { //only render if hand is tracked for (int i = 0; i < 5; i++) { fingerObjectsL[i].GetComponent <Renderer>().enabled = false; fingerObjectsR[i].GetComponent <Renderer>().enabled = false; } wristObjectL.GetComponent <Renderer>().enabled = false; wristObjectR.GetComponent <Renderer>().enabled = false; if (HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbTip, Handedness.Left, out pose)) { tipsL[0] = pose.Position; fingerObjectsL[0].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, Handedness.Left, out pose)) { tipsL[1] = pose.Position; fingerObjectsL[1].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, Handedness.Left, out pose)) { tipsL[2] = pose.Position; fingerObjectsL[2].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, Handedness.Left, out pose)) { tipsL[3] = pose.Position; fingerObjectsL[3].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.PinkyTip, Handedness.Left, out pose)) { tipsL[4] = pose.Position; fingerObjectsL[4].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Handedness.Left, out pose)) { wristL = pose.Position; wristObjectL.GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbTip, Handedness.Right, out pose)) { tipsR[0] = pose.Position; fingerObjectsR[0].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, Handedness.Right, out pose)) { tipsR[1] = pose.Position; fingerObjectsR[1].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, Handedness.Right, out pose)) { tipsR[2] = pose.Position; fingerObjectsR[2].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, Handedness.Right, out pose)) { tipsR[3] = pose.Position; fingerObjectsR[3].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.PinkyTip, Handedness.Right, out pose)) { tipsR[4] = pose.Position; fingerObjectsR[4].GetComponent <Renderer>().enabled = true; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Handedness.Right, out pose)) { wristR = pose.Position; wristObjectR.GetComponent <Renderer>().enabled = true; } for (int i = 0; i < 5; i++) { fingerObjectsL[i].transform.position = tipsL[i] + (tipsL[i] - wristL) * dist; fingerObjectsR[i].transform.position = tipsR[i] + (tipsR[i] - wristR) * dist; wristObjectL.transform.position = wristL; wristObjectR.transform.position = wristR; } }
void Update() { if (!photonView.IsMine) { if (_isShow) { if (((clientCounterFrames % clientUpdateFrameRate) == 0)) { //now the wrist. lerp motion and rotation GameObject wrist = GetJointObject(TrackedHandJoint.Wrist); float lerpTime = Mathf.Clamp(InterpolatePointLerpSpeed * Time.smoothDeltaTime, 0.33f, 1.0f); wrist.transform.localPosition = Vector3.Lerp(wrist.transform.localPosition, _rootPos, lerpTime); Vector3 rotV = wrist.transform.localEulerAngles; wrist.transform.localEulerAngles = Vector3.Lerp(rotV, _rootRot.Normalise(rotV - new Vector3(180, 180, 180), rotV + new Vector3(180, 180, 180)), lerpTime); GameObject objectJoint; float x, y, z, w; for (int key = 0; key < _jointNameListArray.Length; key++) { int index = (int)_jointNameListArray[key]; if (index >= 0 && index < jointObjects.Length) { if (index == (int)TrackedHandJoint.Wrist) { continue; } objectJoint = jointObjects[index]; if (objectJoint != null) { int rIdx = (int)index * 4; x = (float)ShortToFloat(_rotArrayShorts[rIdx + 0]); y = (float)ShortToFloat(_rotArrayShorts[rIdx + 1]); z = (float)ShortToFloat(_rotArrayShorts[rIdx + 2]); w = (float)ShortToFloat(_rotArrayShorts[rIdx + 3]); //lerp if (x == 0 && y == 0 && z == 0 && w == 0) { //zero quaternions are invalid //Debug.Log("Zero Quat: x==0&&y==0&&z==0&&w==0"); continue; } Quaternion rotQ = objectJoint.transform.localRotation; objectJoint.transform.localRotation = Quaternion.Lerp(rotQ, new Quaternion(x, y, z, w).normalized, lerpTime); } } } clientCounterFrames = 0; } ++clientCounterFrames; } } else { bool isHandExist = HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Hand, out MixedRealityPose pose); if (isHandExist) { if (_mixedRealityHand == null) { _mixedRealityHand = HandJointUtils.FindHand(Hand); } if (((localCounterFrames % localUpdateFrameRate) == 0)) { SetJoint(TrackedHandJoint.Wrist, true); // Enable the mesh if its disabled if (!_isShow) { photonView.RPC("setMeshEnabled", RpcTarget.All, true); } // Set each of the joints int count = _jointNameListArray.Length; for (int i = 0; i < count; i++) { if (_jointNameListArray[i] != TrackedHandJoint.Wrist) { SetJoint(_jointNameListArray[i], false); } } } if ((localCounterFrames % clientSendFrameRate) == 0) { // Send the client local joints GameObject wrist = GetJointObject(TrackedHandJoint.Wrist); photonView.SendToAllUnReliableRPC(RPCRuntimeUPDShortcut, ReceiverGroup.Others, wrist.transform.localPosition, wrist.transform.localRotation, _rotArrayShorts); localCounterFrames = 0; } } else if (!isHandExist) { // Disable the mesh if the Hand Wrist doesn't exist _mixedRealityHand = null; if (_isShow) { photonView.RPC("setMeshEnabled", RpcTarget.All, false); } } //always run the counter so its ready to send on first frame when the hand exists ++localCounterFrames; } }
void Update() { if (!active) { pinchSlider.SetActive(true); pinchSliderHor.SetActive(true); handPlane.SetActive(false); return; } if (++curInterval < interval) { return; } curInterval = 0; if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexKnuckle, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po1) && HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po2) && HandJointUtils.TryGetJointPose(TrackedHandJoint.PinkyKnuckle, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po3) && HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbTip, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po4) && HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbProximalJoint, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po5)) { pinchSlider.SetActive(false); pinchSliderHor.SetActive(false); handPlane.SetActive(true); var p1 = ct.TransformWorldCoords(po1.Position); var p2 = ct.TransformWorldCoords(po2.Position); var p3 = ct.TransformWorldCoords(po3.Position); var p4 = ct.TransformWorldCoords(po4.Position); var p5 = ct.TransformWorldCoords(po5.Position); /* * float deltaX = p4.x - p1.x; * float deltaY = p4.y - p1.y; * float deltaZ = p4.z - p1.z; * float distance = (float)System.Math.Sqrt(deltaX * deltaX + deltaY * deltaY + deltaZ * deltaZ); * log.saySomething("distance: " + distance); */ float angle = Vector3.Angle(p4 - p5, p1 - p5); if (//angle > 70 || p1.x < -0.5 || 0.5 < p1.x || p1.y < -0.5 || 0.5 < p1.y || p1.z < -0.5 || 0.5 < p1.z || p2.x < -0.5 || 0.5 < p2.x || p2.y < -0.5 || 0.5 < p2.y || p2.z < -0.5 || 0.5 < p2.z || p3.x < -0.5 || 0.5 < p3.x || p3.y < -0.5 || 0.5 < p3.y || p3.z < -0.5 || 0.5 < p3.z) { return; } locked = angle > 60; if (!locked) { plane = leftHanded ? new Plane(p1, p3, p2) : new Plane(p1, p2, p3); } else { plane = new Plane(plane.normal.normalized, p1); } var orig = plane.ClosestPointOnPlane(Vector3.zero); var dy = (p2 - p1).normalized; var dx = Vector3.Cross(dy, plane.normal); if (enableReferencePlane) { if (!referencePlane) { referencePlane = GameObject.CreatePrimitive(PrimitiveType.Plane); referencePlane.GetComponent <Transform>().SetParent(ct.gameObject.GetComponent <Transform>()); referencePlane.GetComponent <Transform>().localScale = new Vector3(0.02f, 0.02f, 0.02f); } referencePlane.GetComponent <Transform>().up = plane.normal; referencePlane.GetComponent <Transform>().localPosition = p2; } ct.Slice(orig, dx, dy, tex); } }
void LeftHand() { if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, myHandedness, out MixedRealityPose palmPose)) { if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, leftHandedness, out MixedRealityPose poseL)) { // ... if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, leftHandedness, out MixedRealityPose midPoseL)) { // ... if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, leftHandedness, out MixedRealityPose pointerPoseL)) { // ... if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, leftHandedness, out MixedRealityPose palmPoseL)) { HandEvents l_handEvent = new HandEvents(poseL.Position, Time.time); l_indexHandEvents.Add(l_handEvent); if (l_indexHandEvents.Count >= maxSize) { l_indexHandEvents.RemoveAt(0); } HandEvents l_midHandEvent = new HandEvents(midPoseL.Position, Time.time); l_middleHandEvents.Add(l_midHandEvent); if (l_middleHandEvents.Count >= maxSize) { l_middleHandEvents.RemoveAt(0); } HandEvents l_pointerHandEvent = new HandEvents(pointerPoseL.Position, Time.time); l_ringHandEvents.Add(l_pointerHandEvent); if (l_ringHandEvents.Count >= maxSize) { l_ringHandEvents.RemoveAt(0); } HandEvents l_palmHandEvent = new HandEvents(palmPoseL.Position, Time.time); l_palmHandEvents.Add(l_palmHandEvent); if (l_palmHandEvents.Count >= maxSize) { l_palmHandEvents.RemoveAt(0); } palmPointer.transform.position = palmPose.Position; indexPointerL.transform.position = poseL.Position; middlePointerL.transform.position = midPoseL.Position; ringPointerL.transform.position = pointerPoseL.Position; palmPointerL.transform.position = palmPoseL.Position; //Get distance from palm to palm float palmToPalm = Vector3.Distance(palmHandEvents[0].position, l_palmHandEvents[0].position); float palmToPalm2 = Vector3.Distance(palmHandEvents[palmHandEvents.Count - 1].position, l_palmHandEvents[l_palmHandEvents.Count - 1].position); indexTextL.text = (Mathf.Round(palmToPalm2 * 1000f) / 1000).ToString(); ringTextL.text = (Mathf.Round(palmToPalm * 1000f) / 1000).ToString(); var indexRendererL = indexPointerL.GetComponent <Renderer>(); var middleRendererL = middlePointerL.GetComponent <Renderer>(); var ringRendererL = ringPointerL.GetComponent <Renderer>(); var palmRendererL = palmPointerL.GetComponent <Renderer>(); //Hands comming together if (palmToPalm <= .05f && palmToPalm2 <= 0.05f) { float currTime = Time.time; if (currTime - lastCommand2 > delay) { speechCommands.ChangeColor(); } lastCommand2 = currTime; indexRendererL.material.SetColor("_Color", Color.red); middleRendererL.material.SetColor("_Color", Color.red); ringRendererL.material.SetColor("_Color", Color.red); palmRendererL.material.SetColor("_Color", Color.red); } else { float currTime = Time.time; if (currTime - lastCommand2 > delay) { indexRendererL.material.SetColor("_Color", Color.blue); middleRendererL.material.SetColor("_Color", Color.blue); ringRendererL.material.SetColor("_Color", Color.blue); palmRendererL.material.SetColor("_Color", Color.blue); } } } } } } } }
void RightHand() { if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, myHandedness, out MixedRealityPose pose)) { // ... if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, myHandedness, out MixedRealityPose midPose)) { // ... if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, myHandedness, out MixedRealityPose pointerPose)) { // ... if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, myHandedness, out MixedRealityPose palmPose)) { HandEvents handEvent = new HandEvents(pose.Position, Time.time); indexHandEvents.Add(handEvent); if (indexHandEvents.Count >= maxSize) { indexHandEvents.RemoveAt(0); } HandEvents midHandEvent = new HandEvents(midPose.Position, Time.time); middleHandEvents.Add(midHandEvent); if (middleHandEvents.Count >= maxSize) { middleHandEvents.RemoveAt(0); } HandEvents pointerHandEvent = new HandEvents(pointerPose.Position, Time.time); ringHandEvents.Add(pointerHandEvent); if (ringHandEvents.Count >= maxSize) { ringHandEvents.RemoveAt(0); } HandEvents palmHandEvent = new HandEvents(palmPose.Position, Time.time); palmHandEvents.Add(palmHandEvent); if (palmHandEvents.Count >= maxSize) { palmHandEvents.RemoveAt(0); } indexPointer.transform.position = pose.Position; middlePointer.transform.position = midPose.Position; ringPointer.transform.position = pointerPose.Position; palmPointer.transform.position = palmPose.Position; //Get average distance to palm of first (right) float distIndexPalmR = Vector3.Distance(indexHandEvents[0].position, palmHandEvents[0].position); float distMidPalmR = Vector3.Distance(middleHandEvents[0].position, palmHandEvents[0].position); float distRingPalmR = Vector3.Distance(ringHandEvents[0].position, palmHandEvents[0].position); float avgFirst = (distIndexPalmR + distMidPalmR + distRingPalmR) / 3f; float avgPointFirst = (distMidPalmR + distRingPalmR) / 2f; //Get average distance of palm of last (right) float dist2IndexPalmR = Vector3.Distance(indexHandEvents[indexHandEvents.Count - 1].position, palmHandEvents[palmHandEvents.Count - 1].position); float dist2MidPalmR = Vector3.Distance(middleHandEvents[middleHandEvents.Count - 1].position, palmHandEvents[palmHandEvents.Count - 1].position); float dist2RingPalmR = Vector3.Distance(ringHandEvents[ringHandEvents.Count - 1].position, palmHandEvents[palmHandEvents.Count - 1].position); float avgSecond = (dist2IndexPalmR + dist2MidPalmR + dist2RingPalmR) / 3f; float avgPointSecond = (dist2MidPalmR + dist2RingPalmR) / 2f; indexText.text = (Mathf.Round(avgSecond * 1000f) / 1000).ToString(); ringText.text = (Mathf.Round(avgFirst * 1000f) / 1000).ToString(); var indexRenderer = indexPointer.GetComponent <Renderer>(); var middleRenderer = middlePointer.GetComponent <Renderer>(); var ringRenderer = ringPointer.GetComponent <Renderer>(); var palmRenderer = palmPointer.GetComponent <Renderer>(); //Right hand check if hand becomes a fist if (avgSecond <= avgFirst / 1.7 && avgFirst >= .03) { float currTime = Time.time; if (currTime - lastCommand > delay) { returnGesture = true; //speechCommands.Return(); } lastCommand = currTime; //var indexRenderer = indexPointer.GetComponent<Renderer>(); indexRenderer.material.SetColor("_Color", Color.white); middleRenderer.material.SetColor("_Color", Color.white); ringRenderer.material.SetColor("_Color", Color.white); palmRenderer.material.SetColor("_Color", Color.white); } else { float currTime = Time.time; if (currTime - lastCommand > delay) { returnGesture = false; //var indexRenderer = indexPointer.GetComponent<Renderer>(); indexRenderer.material.SetColor("_Color", Color.black); middleRenderer.material.SetColor("_Color", Color.black); ringRenderer.material.SetColor("_Color", Color.black); palmRenderer.material.SetColor("_Color", Color.black); } } //Check if pointing if (dist2IndexPalmR >= avgPointSecond * 1.2 && distIndexPalmR >= avgPointFirst * 1.2 && avgPointSecond <= .03 && avgPointFirst <= 0.03) { float currTime = Time.time; if (currTime - lastCommand3 > delay) { pointGesture = true; //speechCommands.Point(); } lastCommand = currTime; //var indexRenderer = indexPointer.GetComponent<Renderer>(); indexRenderer.material.SetColor("_Color", Color.magenta); middleRenderer.material.SetColor("_Color", Color.magenta); ringRenderer.material.SetColor("_Color", Color.magenta); palmRenderer.material.SetColor("_Color", Color.magenta); } else { float currTime = Time.time; if (currTime - lastCommand > delay) { pointGesture = false; //var indexRenderer = indexPointer.GetComponent<Renderer>(); indexRenderer.material.SetColor("_Color", Color.black); middleRenderer.material.SetColor("_Color", Color.black); ringRenderer.material.SetColor("_Color", Color.black); palmRenderer.material.SetColor("_Color", Color.black); } } if (pointGesture) { speechCommands.Point(); } else { speechCommands.NotPoint(); if (returnGesture) { speechCommands.Come(); } } } } } } }
public static void ReceiveMessage(float[] points) { frame++; float timeElapsed = Time.time - timeSinceStartOfRecording; timesteps[frame] = timeElapsed; int size = points.Length / 3; Vector3[] pcl_temp = new Vector3[size]; Debug.Log(size); // get mrtk joints Vector3 leftPalm = new Vector3(); Vector3 rightPalm = new Vector3(); if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, Handedness.Left, out MixedRealityPose leftPose)) { leftPalm = leftPose.Position; } if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, Handedness.Right, out MixedRealityPose rightPose)) { rightPalm = rightPose.Position; } // append null palm coordinates and check later //leftPalmArray[frame_count] = leftPalm; //rightPalmArray[frame_count] = rightPalm; if (leftPalm == null) { leftPalm = new Vector3(0, 0, 0); } if (rightPalm == null) { rightPalm = new Vector3(0, 0, 0); } float THRESH = 0.2F; int n_points = 0; bool[] filter = new bool[size]; for (int n = 0; n < size; n++) { float min_base = float.PositiveInfinity; Vector3 pt = new Vector3(points[n * 3], points[n * 3 + 1], points[n * 3 + 2]); pcl_temp[n] = pt; float dist1 = Vector3.Distance(pt, leftPalm); float dist2 = Vector3.Distance(pt, rightPalm); float m = Math.Min(min_base, Math.Min(dist1, dist2)); if (m < THRESH) { n_points++; filter[n] = true; } else { filter[n] = false; } } pcl = new Vector3[n_points]; int k = 0; for (int n = 0; n < size; n++) { if (filter[n]) { pcl[k] = pcl_temp[n]; k++; } } Pairs[frame] = pcl; Debug.Log("Subscriber: point cloud data received"); }