void Update()
{
//only render if hand is tracked
for (int i = 0; i < 5; i++)
{
fingerObjectsL[i].GetComponent <Renderer>().enabled = false;
fingerObjectsR[i].GetComponent <Renderer>().enabled = false;
}
wristObjectL.GetComponent <Renderer>().enabled = false;
wristObjectR.GetComponent <Renderer>().enabled = false;
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbTip, Handedness.Left, out pose))
{
tipsL[0] = pose.Position;
fingerObjectsL[0].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, Handedness.Left, out pose))
{
tipsL[1] = pose.Position;
fingerObjectsL[1].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, Handedness.Left, out pose))
{
tipsL[2] = pose.Position;
fingerObjectsL[2].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, Handedness.Left, out pose))
{
tipsL[3] = pose.Position;
fingerObjectsL[3].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.PinkyTip, Handedness.Left, out pose))
{
tipsL[4] = pose.Position;
fingerObjectsL[4].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Handedness.Left, out pose))
{
wristL = pose.Position;
wristObjectL.GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbTip, Handedness.Right, out pose))
{
tipsR[0] = pose.Position;
fingerObjectsR[0].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, Handedness.Right, out pose))
{
tipsR[1] = pose.Position;
fingerObjectsR[1].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, Handedness.Right, out pose))
{
tipsR[2] = pose.Position;
fingerObjectsR[2].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, Handedness.Right, out pose))
{
tipsR[3] = pose.Position;
fingerObjectsR[3].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.PinkyTip, Handedness.Right, out pose))
{
tipsR[4] = pose.Position;
fingerObjectsR[4].GetComponent <Renderer>().enabled = true;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Handedness.Right, out pose))
{
wristR = pose.Position;
wristObjectR.GetComponent <Renderer>().enabled = true;
}
for (int i = 0; i < 5; i++)
{
fingerObjectsL[i].transform.position = tipsL[i] + (tipsL[i] - wristL) * dist;
fingerObjectsR[i].transform.position = tipsR[i] + (tipsR[i] - wristR) * dist;
wristObjectL.transform.position = wristL;
wristObjectR.transform.position = wristR;
}
}
void Update()
{
if (!photonView.IsMine)
{
if (_isShow)
{
if (((clientCounterFrames % clientUpdateFrameRate) == 0))
{
//now the wrist. lerp motion and rotation
GameObject wrist = GetJointObject(TrackedHandJoint.Wrist);
float lerpTime = Mathf.Clamp(InterpolatePointLerpSpeed * Time.smoothDeltaTime, 0.33f, 1.0f);
wrist.transform.localPosition = Vector3.Lerp(wrist.transform.localPosition, _rootPos, lerpTime);
Vector3 rotV = wrist.transform.localEulerAngles;
wrist.transform.localEulerAngles = Vector3.Lerp(rotV, _rootRot.Normalise(rotV - new Vector3(180, 180, 180), rotV + new Vector3(180, 180, 180)), lerpTime);
GameObject objectJoint;
float x, y, z, w;
for (int key = 0; key < _jointNameListArray.Length; key++)
{
int index = (int)_jointNameListArray[key];
if (index >= 0 && index < jointObjects.Length)
{
if (index == (int)TrackedHandJoint.Wrist)
{
continue;
}
objectJoint = jointObjects[index];
if (objectJoint != null)
{
int rIdx = (int)index * 4;
x = (float)ShortToFloat(_rotArrayShorts[rIdx + 0]);
y = (float)ShortToFloat(_rotArrayShorts[rIdx + 1]);
z = (float)ShortToFloat(_rotArrayShorts[rIdx + 2]);
w = (float)ShortToFloat(_rotArrayShorts[rIdx + 3]);
//lerp
if (x == 0 && y == 0 && z == 0 && w == 0)
{
//zero quaternions are invalid
//Debug.Log("Zero Quat: x==0&&y==0&&z==0&&w==0");
continue;
}
Quaternion rotQ = objectJoint.transform.localRotation;
objectJoint.transform.localRotation = Quaternion.Lerp(rotQ, new Quaternion(x, y, z, w).normalized, lerpTime);
}
}
}
clientCounterFrames = 0;
}
++clientCounterFrames;
}
}
else
{
bool isHandExist = HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Hand, out MixedRealityPose pose);
if (isHandExist)
{
if (_mixedRealityHand == null)
{
_mixedRealityHand = HandJointUtils.FindHand(Hand);
}
if (((localCounterFrames % localUpdateFrameRate) == 0))
{
SetJoint(TrackedHandJoint.Wrist, true);
// Enable the mesh if its disabled
if (!_isShow)
{
photonView.RPC("setMeshEnabled", RpcTarget.All, true);
}
// Set each of the joints
int count = _jointNameListArray.Length;
for (int i = 0; i < count; i++)
{
if (_jointNameListArray[i] != TrackedHandJoint.Wrist)
{
SetJoint(_jointNameListArray[i], false);
}
}
}
if ((localCounterFrames % clientSendFrameRate) == 0)
{
// Send the client local joints
GameObject wrist = GetJointObject(TrackedHandJoint.Wrist);
photonView.SendToAllUnReliableRPC(RPCRuntimeUPDShortcut, ReceiverGroup.Others, wrist.transform.localPosition, wrist.transform.localRotation, _rotArrayShorts);
localCounterFrames = 0;
}
}
else if (!isHandExist)
{
// Disable the mesh if the Hand Wrist doesn't exist
_mixedRealityHand = null;
if (_isShow)
{
photonView.RPC("setMeshEnabled", RpcTarget.All, false);
}
}
//always run the counter so its ready to send on first frame when the hand exists
++localCounterFrames;
}
}
void Update()
{
if (!active)
{
pinchSlider.SetActive(true);
pinchSliderHor.SetActive(true);
handPlane.SetActive(false);
return;
}
if (++curInterval < interval)
{
return;
}
curInterval = 0;
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexKnuckle, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po1) &&
HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po2) &&
HandJointUtils.TryGetJointPose(TrackedHandJoint.PinkyKnuckle, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po3) &&
HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbTip, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po4) &&
HandJointUtils.TryGetJointPose(TrackedHandJoint.ThumbProximalJoint, leftHanded ? Handedness.Left : Handedness.Right, out MixedRealityPose po5))
{
pinchSlider.SetActive(false);
pinchSliderHor.SetActive(false);
handPlane.SetActive(true);
var p1 = ct.TransformWorldCoords(po1.Position);
var p2 = ct.TransformWorldCoords(po2.Position);
var p3 = ct.TransformWorldCoords(po3.Position);
var p4 = ct.TransformWorldCoords(po4.Position);
var p5 = ct.TransformWorldCoords(po5.Position);
/*
* float deltaX = p4.x - p1.x;
* float deltaY = p4.y - p1.y;
* float deltaZ = p4.z - p1.z;
* float distance = (float)System.Math.Sqrt(deltaX * deltaX + deltaY * deltaY + deltaZ * deltaZ);
* log.saySomething("distance: " + distance);
*/
float angle = Vector3.Angle(p4 - p5, p1 - p5);
if (//angle > 70 ||
p1.x < -0.5 || 0.5 < p1.x ||
p1.y < -0.5 || 0.5 < p1.y ||
p1.z < -0.5 || 0.5 < p1.z ||
p2.x < -0.5 || 0.5 < p2.x ||
p2.y < -0.5 || 0.5 < p2.y ||
p2.z < -0.5 || 0.5 < p2.z ||
p3.x < -0.5 || 0.5 < p3.x ||
p3.y < -0.5 || 0.5 < p3.y ||
p3.z < -0.5 || 0.5 < p3.z)
{
return;
}
locked = angle > 60;
if (!locked)
{
plane = leftHanded ? new Plane(p1, p3, p2) : new Plane(p1, p2, p3);
}
else
{
plane = new Plane(plane.normal.normalized, p1);
}
var orig = plane.ClosestPointOnPlane(Vector3.zero);
var dy = (p2 - p1).normalized;
var dx = Vector3.Cross(dy, plane.normal);
if (enableReferencePlane)
{
if (!referencePlane)
{
referencePlane = GameObject.CreatePrimitive(PrimitiveType.Plane);
referencePlane.GetComponent <Transform>().SetParent(ct.gameObject.GetComponent <Transform>());
referencePlane.GetComponent <Transform>().localScale = new Vector3(0.02f, 0.02f, 0.02f);
}
referencePlane.GetComponent <Transform>().up = plane.normal;
referencePlane.GetComponent <Transform>().localPosition = p2;
}
ct.Slice(orig, dx, dy, tex);
}
}
void LeftHand()
{
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, myHandedness, out MixedRealityPose palmPose))
{
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, leftHandedness, out MixedRealityPose poseL))
{
// ...
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, leftHandedness, out MixedRealityPose midPoseL))
{
// ...
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, leftHandedness, out MixedRealityPose pointerPoseL))
{
// ...
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, leftHandedness, out MixedRealityPose palmPoseL))
{
HandEvents l_handEvent = new HandEvents(poseL.Position, Time.time);
l_indexHandEvents.Add(l_handEvent);
if (l_indexHandEvents.Count >= maxSize)
{
l_indexHandEvents.RemoveAt(0);
}
HandEvents l_midHandEvent = new HandEvents(midPoseL.Position, Time.time);
l_middleHandEvents.Add(l_midHandEvent);
if (l_middleHandEvents.Count >= maxSize)
{
l_middleHandEvents.RemoveAt(0);
}
HandEvents l_pointerHandEvent = new HandEvents(pointerPoseL.Position, Time.time);
l_ringHandEvents.Add(l_pointerHandEvent);
if (l_ringHandEvents.Count >= maxSize)
{
l_ringHandEvents.RemoveAt(0);
}
HandEvents l_palmHandEvent = new HandEvents(palmPoseL.Position, Time.time);
l_palmHandEvents.Add(l_palmHandEvent);
if (l_palmHandEvents.Count >= maxSize)
{
l_palmHandEvents.RemoveAt(0);
}
palmPointer.transform.position = palmPose.Position;
indexPointerL.transform.position = poseL.Position;
middlePointerL.transform.position = midPoseL.Position;
ringPointerL.transform.position = pointerPoseL.Position;
palmPointerL.transform.position = palmPoseL.Position;
//Get distance from palm to palm
float palmToPalm = Vector3.Distance(palmHandEvents[0].position, l_palmHandEvents[0].position);
float palmToPalm2 = Vector3.Distance(palmHandEvents[palmHandEvents.Count - 1].position, l_palmHandEvents[l_palmHandEvents.Count - 1].position);
indexTextL.text = (Mathf.Round(palmToPalm2 * 1000f) / 1000).ToString();
ringTextL.text = (Mathf.Round(palmToPalm * 1000f) / 1000).ToString();
var indexRendererL = indexPointerL.GetComponent <Renderer>();
var middleRendererL = middlePointerL.GetComponent <Renderer>();
var ringRendererL = ringPointerL.GetComponent <Renderer>();
var palmRendererL = palmPointerL.GetComponent <Renderer>();
//Hands comming together
if (palmToPalm <= .05f && palmToPalm2 <= 0.05f)
{
float currTime = Time.time;
if (currTime - lastCommand2 > delay)
{
speechCommands.ChangeColor();
}
lastCommand2 = currTime;
indexRendererL.material.SetColor("_Color", Color.red);
middleRendererL.material.SetColor("_Color", Color.red);
ringRendererL.material.SetColor("_Color", Color.red);
palmRendererL.material.SetColor("_Color", Color.red);
}
else
{
float currTime = Time.time;
if (currTime - lastCommand2 > delay)
{
indexRendererL.material.SetColor("_Color", Color.blue);
middleRendererL.material.SetColor("_Color", Color.blue);
ringRendererL.material.SetColor("_Color", Color.blue);
palmRendererL.material.SetColor("_Color", Color.blue);
}
}
}
}
}
}
}
}
void RightHand()
{
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.IndexTip, myHandedness, out MixedRealityPose pose))
{
// ...
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.MiddleTip, myHandedness, out MixedRealityPose midPose))
{
// ...
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.RingTip, myHandedness, out MixedRealityPose pointerPose))
{
// ...
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, myHandedness, out MixedRealityPose palmPose))
{
HandEvents handEvent = new HandEvents(pose.Position, Time.time);
indexHandEvents.Add(handEvent);
if (indexHandEvents.Count >= maxSize)
{
indexHandEvents.RemoveAt(0);
}
HandEvents midHandEvent = new HandEvents(midPose.Position, Time.time);
middleHandEvents.Add(midHandEvent);
if (middleHandEvents.Count >= maxSize)
{
middleHandEvents.RemoveAt(0);
}
HandEvents pointerHandEvent = new HandEvents(pointerPose.Position, Time.time);
ringHandEvents.Add(pointerHandEvent);
if (ringHandEvents.Count >= maxSize)
{
ringHandEvents.RemoveAt(0);
}
HandEvents palmHandEvent = new HandEvents(palmPose.Position, Time.time);
palmHandEvents.Add(palmHandEvent);
if (palmHandEvents.Count >= maxSize)
{
palmHandEvents.RemoveAt(0);
}
indexPointer.transform.position = pose.Position;
middlePointer.transform.position = midPose.Position;
ringPointer.transform.position = pointerPose.Position;
palmPointer.transform.position = palmPose.Position;
//Get average distance to palm of first (right)
float distIndexPalmR = Vector3.Distance(indexHandEvents[0].position, palmHandEvents[0].position);
float distMidPalmR = Vector3.Distance(middleHandEvents[0].position, palmHandEvents[0].position);
float distRingPalmR = Vector3.Distance(ringHandEvents[0].position, palmHandEvents[0].position);
float avgFirst = (distIndexPalmR + distMidPalmR + distRingPalmR) / 3f;
float avgPointFirst = (distMidPalmR + distRingPalmR) / 2f;
//Get average distance of palm of last (right)
float dist2IndexPalmR = Vector3.Distance(indexHandEvents[indexHandEvents.Count - 1].position, palmHandEvents[palmHandEvents.Count - 1].position);
float dist2MidPalmR = Vector3.Distance(middleHandEvents[middleHandEvents.Count - 1].position, palmHandEvents[palmHandEvents.Count - 1].position);
float dist2RingPalmR = Vector3.Distance(ringHandEvents[ringHandEvents.Count - 1].position, palmHandEvents[palmHandEvents.Count - 1].position);
float avgSecond = (dist2IndexPalmR + dist2MidPalmR + dist2RingPalmR) / 3f;
float avgPointSecond = (dist2MidPalmR + dist2RingPalmR) / 2f;
indexText.text = (Mathf.Round(avgSecond * 1000f) / 1000).ToString();
ringText.text = (Mathf.Round(avgFirst * 1000f) / 1000).ToString();
var indexRenderer = indexPointer.GetComponent <Renderer>();
var middleRenderer = middlePointer.GetComponent <Renderer>();
var ringRenderer = ringPointer.GetComponent <Renderer>();
var palmRenderer = palmPointer.GetComponent <Renderer>();
//Right hand check if hand becomes a fist
if (avgSecond <= avgFirst / 1.7 && avgFirst >= .03)
{
float currTime = Time.time;
if (currTime - lastCommand > delay)
{
returnGesture = true;
//speechCommands.Return();
}
lastCommand = currTime;
//var indexRenderer = indexPointer.GetComponent<Renderer>();
indexRenderer.material.SetColor("_Color", Color.white);
middleRenderer.material.SetColor("_Color", Color.white);
ringRenderer.material.SetColor("_Color", Color.white);
palmRenderer.material.SetColor("_Color", Color.white);
}
else
{
float currTime = Time.time;
if (currTime - lastCommand > delay)
{
returnGesture = false;
//var indexRenderer = indexPointer.GetComponent<Renderer>();
indexRenderer.material.SetColor("_Color", Color.black);
middleRenderer.material.SetColor("_Color", Color.black);
ringRenderer.material.SetColor("_Color", Color.black);
palmRenderer.material.SetColor("_Color", Color.black);
}
}
//Check if pointing
if (dist2IndexPalmR >= avgPointSecond * 1.2 && distIndexPalmR >= avgPointFirst * 1.2 && avgPointSecond <= .03 && avgPointFirst <= 0.03)
{
float currTime = Time.time;
if (currTime - lastCommand3 > delay)
{
pointGesture = true;
//speechCommands.Point();
}
lastCommand = currTime;
//var indexRenderer = indexPointer.GetComponent<Renderer>();
indexRenderer.material.SetColor("_Color", Color.magenta);
middleRenderer.material.SetColor("_Color", Color.magenta);
ringRenderer.material.SetColor("_Color", Color.magenta);
palmRenderer.material.SetColor("_Color", Color.magenta);
}
else
{
float currTime = Time.time;
if (currTime - lastCommand > delay)
{
pointGesture = false;
//var indexRenderer = indexPointer.GetComponent<Renderer>();
indexRenderer.material.SetColor("_Color", Color.black);
middleRenderer.material.SetColor("_Color", Color.black);
ringRenderer.material.SetColor("_Color", Color.black);
palmRenderer.material.SetColor("_Color", Color.black);
}
}
if (pointGesture)
{
speechCommands.Point();
}
else
{
speechCommands.NotPoint();
if (returnGesture)
{
speechCommands.Come();
}
}
}
}
}
}
}
public static void ReceiveMessage(float[] points)
{
frame++;
float timeElapsed = Time.time - timeSinceStartOfRecording;
timesteps[frame] = timeElapsed;
int size = points.Length / 3;
Vector3[] pcl_temp = new Vector3[size];
Debug.Log(size);
// get mrtk joints
Vector3 leftPalm = new Vector3();
Vector3 rightPalm = new Vector3();
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, Handedness.Left, out MixedRealityPose leftPose))
{
leftPalm = leftPose.Position;
}
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Palm, Handedness.Right, out MixedRealityPose rightPose))
{
rightPalm = rightPose.Position;
}
// append null palm coordinates and check later
//leftPalmArray[frame_count] = leftPalm;
//rightPalmArray[frame_count] = rightPalm;
if (leftPalm == null)
{
leftPalm = new Vector3(0, 0, 0);
}
if (rightPalm == null)
{
rightPalm = new Vector3(0, 0, 0);
}
float THRESH = 0.2F;
int n_points = 0;
bool[] filter = new bool[size];
for (int n = 0; n < size; n++)
{
float min_base = float.PositiveInfinity;
Vector3 pt = new Vector3(points[n * 3], points[n * 3 + 1], points[n * 3 + 2]);
pcl_temp[n] = pt;
float dist1 = Vector3.Distance(pt, leftPalm);
float dist2 = Vector3.Distance(pt, rightPalm);
float m = Math.Min(min_base, Math.Min(dist1, dist2));
if (m < THRESH)
{
n_points++;
filter[n] = true;
}
else
{
filter[n] = false;
}
}
pcl = new Vector3[n_points];
int k = 0;
for (int n = 0; n < size; n++)
{
if (filter[n])
{
pcl[k] = pcl_temp[n];
k++;
}
}
Pairs[frame] = pcl;
Debug.Log("Subscriber: point cloud data received");
}
