/// <inheritdoc />
public override void SolverUpdate()
{
// Determine the new active hand.
IMixedRealityHand newActivehand = null;
foreach (var hand in handStack)
{
if (IsHandActive(hand))
{
newActivehand = hand;
break;
}
}
// Track the new active hand.
if (trackedHand == null || trackedHand != newActivehand)
{
ChangeTrackedObjectType(newActivehand);
}
// Update the goal position.
GoalPosition = CalculateGoalPosition();
GoalRotation = billboardToCamera ? Quaternion.LookRotation(CameraCache.Main.transform.forward) :
SolverHandler.TransformTarget.rotation;
if (trackedHand != null)
{
UpdateWorkingPositionToGoal();
UpdateWorkingRotationToGoal();
}
}
public void Update()
{
if (HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Handedness.Any, out MixedRealityPose pose))
{
Vector3 positionOfWrist = pose.Position;
IMixedRealityHand whichHand = HandJointUtils.FindHand(Handedness.Any);
}
}
public void OnSourceLost(SourceStateEventData eventData)
{
// Only react to articulated hand input sources
if (hand != null)
{
handsText.GetComponent <TextMeshPro>().text = "Source lost: " + hand.ControllerHandedness;
}
hand = null;
}
protected void Start()
{
handBounds = GetComponent <HandBounds>();
// Initially no hands are tacked or active.
trackedHand = null;
onLastHandLost.Invoke();
onHandDeactivate.Invoke();
}
public void OnSourceDetected(SourceStateEventData eventData)
{
IMixedRealityHand hand = eventData.Controller as IMixedRealityHand;
if (hand != null)
{
if (hand.TryGetJoint(TrackedHandJoint.Wrist, out MixedRealityPose pose))
{
Vector3 positionOfWrist = pose.Position;
}
}
}
/// <summary>
/// Try to find the first matching hand controller and return the pose of the requested joint for that hand.
/// </summary>
public static bool TryGetJointPose(TrackedHandJoint joint, Handedness handedness, out MixedRealityPose pose)
{
IMixedRealityHand hand = FindHand(handedness);
if (hand != null)
{
return(hand.TryGetJoint(joint, out pose));
}
pose = MixedRealityPose.ZeroIdentity;
return(false);
}
void LateUpdate()
{
IMixedRealityHand hand = GetController(trackedHandedness) as IMixedRealityHand;
if (hand == null || !hand.TryGetJoint(trackedJoint, out MixedRealityPose pose))
{
SetChildrenActive(false);
return;
}
SetChildrenActive(true);
transform.position = pose.Position;
transform.rotation = pose.Rotation;
}
/// <inheritdoc />
public override void LateUpdate()
{
base.LateUpdate();
leftHand = null;
rightHand = null;
foreach (var detectedController in Service.DetectedControllers)
{
var hand = detectedController as IMixedRealityHand;
if (hand != null)
{
if (detectedController.ControllerHandedness == Handedness.Left)
{
if (leftHand == null)
{
leftHand = hand;
}
}
else if (detectedController.ControllerHandedness == Handedness.Right)
{
if (rightHand == null)
{
rightHand = hand;
}
}
}
}
if (leftHand != null)
{
foreach (var fauxJoint in leftHandFauxJoints)
{
if (leftHand.TryGetJoint(fauxJoint.Key, out MixedRealityPose pose))
{
fauxJoint.Value.SetPositionAndRotation(pose.Position, pose.Rotation);
}
}
}
if (rightHand != null)
{
foreach (var fauxJoint in rightHandFauxJoints)
{
if (rightHand.TryGetJoint(fauxJoint.Key, out MixedRealityPose pose))
{
fauxJoint.Value.SetPositionAndRotation(pose.Position, pose.Rotation);
}
}
}
}
/// <summary>
/// Determines if a hand meets the requirements for use with constraining the tracked object and determines if the
/// palm is currently facing the user.
/// </summary>
/// <param name="hand">The hand to check against.</param>
/// <returns>True if this hand should be used from tracking.</returns>
protected override bool IsHandActive(IMixedRealityHand hand)
{
if (!base.IsHandActive(hand))
{
return(false);
}
MixedRealityPose pose;
if (hand.TryGetJoint(TrackedHandJoint.Palm, out pose))
{
return(Vector3.Angle(pose.Up, CameraCache.Main.transform.forward) < facingThreshold);
}
return(true);
}
private void SetController(IMixedRealityController controller)
{
IMixedRealityHand hand = controller as IMixedRealityHand;
IMixedRealityController mrcontroller = controller as IMixedRealityController;
if (hand != null)
{
try
{
detectedHand = hand;
if (controller.ControllerHandedness == Handedness.Left)
{
leftHand = hand;
}
else if (controller.ControllerHandedness == Handedness.Right)
{
rightHand = hand;
}
}
catch (Exception)
{
//Debug.Log("Missed a hand, retrying");
}
}
else if (mrcontroller != null)
{
try
{
detectedController = mrcontroller;
if (controller.ControllerHandedness == Handedness.Left)
{
leftController = mrcontroller;
}
else if (controller.ControllerHandedness == Handedness.Right)
{
rightController = mrcontroller;
}
}
catch (Exception)
{
//Debug.Log("Missed a controller, retrying");
}
}
}
/// <summary>
/// Swaps out the currently tracked hand while triggered appropriate events.
/// </summary>
/// <param name="hand">Which hand to track now.</param>
private void ChangeTrackedObjectType(IMixedRealityHand hand)
{
if (hand != null)
{
TrackedObjectType trackedObjectType;
if (HandednessToTrackedObjectType(hand.ControllerHandedness, out trackedObjectType))
{
if (SolverHandler.TrackedObjectToReference != trackedObjectType)
{
SolverHandler.TrackedObjectToReference = trackedObjectType;
// Move the currently tracked hand to the top of the stack.
handStack.Remove(hand);
handStack.Insert(0, hand);
}
if (trackedHand == null)
{
trackedHand = hand;
onHandActivate.Invoke();
}
else
{
StartCoroutine(ToggleCursor(true));
trackedHand = hand;
}
// Wait one frame to disable the cursor in case one hasn't been instantiated yet.
StartCoroutine(ToggleCursor(false, true));
}
else
{
Debug.LogWarning("Failed to change the tracked object type because an IMixedRealityHand could not be resolved to a TrackedObjectType.");
}
}
else
{
if (trackedHand != null)
{
StartCoroutine(ToggleCursor(true));
trackedHand = null;
onHandDeactivate.Invoke();
}
}
}
/// <summary>
/// Determines if a hand meets the requirements for use with constraining the tracked object.
/// </summary>
/// <param name="hand">The hand to check against.</param>
/// <returns>True if this hand should be used from tracking.</returns>
protected virtual bool IsHandActive(IMixedRealityHand hand)
{
// If transitioning between hands is not allowed, make sure the TrackedObjectType matches the hand.
if (!transitionBetweenHands)
{
TrackedObjectType trackedObjectType;
if (HandednessToTrackedObjectType(hand.ControllerHandedness, out trackedObjectType))
{
if (trackedObjectType != SolverHandler.TrackedObjectToReference)
{
return(false);
}
}
}
return(true);
}
public bool TryGetSpecificControllerPose(Handedness handedness, TrackedHandJoint handJointToTrack, out Vector3 position, out Quaternion rotation)
{
bool retrieved = false;
position = Vector3.zero;
rotation = Quaternion.identity;
IMixedRealityHand currentHand = detectedHand;
IMixedRealityController currentController = detectedController;
if (handedness == Handedness.Left)
{
currentHand = leftHand;
currentController = leftController;
}
else if (handedness == Handedness.Right)
{
currentHand = rightHand;
currentController = rightController;
}
if (currentHand != null && currentHand.TryGetJoint(handJointToTrack, out handPose))
{
position = handPose.Position;
rotation = handPose.Rotation;
retrieved = true;
}
else if (currentController != null)
{
try
{
position = currentController.InputSource.Pointers[0].Position;
rotation = currentController.InputSource.Pointers[0].Rotation;
retrieved = true;
}
catch (Exception)
{
retrieved = false;
}
}
return(retrieved);
}
// Start is called before the first frame update
void Awake()
{
RPCRuntimeUPDShortcut = UtilityExtentions.GetRPCShortcut("RPCRuntimeReceiveData");
if (RootTransform.root != null)
{
transform.SetParent(RootTransform.root.transform);
}
var tmp = (TrackedHandJoint[])System.Enum.GetValues(typeof(TrackedHandJoint));
_jointNameListArray = new TrackedHandJoint[tmp.Length - 2];
for (int i = 0, j = 0; i < tmp.Length; i++)
{
if ((i != (int)TrackedHandJoint.None) && (i != (int)TrackedHandJoint.Palm))
{
_jointNameListArray[j++] = tmp[i];
}
}
_mixedRealityHand = HandJointUtils.FindHand(Hand);
transform.localPosition = Vector3.zero;
transform.localRotation = Quaternion.identity;
// init the fade state
setMeshEnabled(false);
// force to hide the mesh at the start
_handStateFading.SetState(HandStateFading.FadeState.Hide);
Debug.Assert(localTargetFPS != 0);
Debug.Assert(clientTargetFPS != 0);
localUpdateFrameRate = (int)(systemFPS / (float)localTargetFPS);
clientSendFrameRate = (int)(systemFPS / (float)clientTargetFPS);
clientUpdateFrameRate = (int)(systemFPS / (float)lerpTargetFPS);
if (localUpdateFrameRate == 0 || clientSendFrameRate == 0 || clientUpdateFrameRate == 0)
{
//prevent divide by zero on the modulus divs below
Debug.Log("FATAL Div by Zero! Local: " + localUpdateFrameRate.ToString() + " Send: " + clientSendFrameRate.ToString() + " Client: " + clientUpdateFrameRate.ToString());
}
}
public Transform RequestJointTransform(TrackedHandJoint joint, Handedness handedness)
{
IMixedRealityHand hand = null;
Dictionary <TrackedHandJoint, Transform> fauxJoints = null;
if (handedness == Handedness.Left)
{
hand = leftHand;
fauxJoints = leftHandFauxJoints;
}
else if (handedness == Handedness.Right)
{
hand = rightHand;
fauxJoints = rightHandFauxJoints;
}
else
{
return(null);
}
Transform jointTransform = null;
if (fauxJoints != null && !fauxJoints.TryGetValue(joint, out jointTransform))
{
jointTransform = new GameObject().transform;
// Since this service survives scene loading and unloading, the fauxJoints it manages need to as well.
Object.DontDestroyOnLoad(jointTransform.gameObject);
jointTransform.name = string.Format("Joint Tracker: {1} {0}", joint, handedness);
if (hand != null && hand.TryGetJoint(joint, out MixedRealityPose pose))
{
jointTransform.SetPositionAndRotation(pose.Position, pose.Rotation);
}
fauxJoints.Add(joint, jointTransform);
}
return(jointTransform);
}
private IEnumerator TestTouchableDistances(BaseNearInteractionTouchable touchable, float colliderThickness, GameObject objectDownExpected)
{
Handedness handedness = Handedness.Right;
ArticulatedHandPose.GestureId gesture = ArticulatedHandPose.GestureId.Open;
yield return(PlayModeTestUtilities.ShowHand(handedness, inputSim));
PokePointer pokePointer = null;
IMixedRealityHand hand = HandJointUtils.FindHand(handedness);
Assert.IsNotNull(hand);
foreach (IMixedRealityPointer pointer in hand.InputSource.Pointers)
{
pokePointer = pointer as PokePointer;
if (pokePointer)
{
break;
}
}
Assert.IsNotNull(pokePointer);
float touchableDistance = pokePointer.TouchableDistance;
float debounceThreshold = 0.01f;
touchable.DebounceThreshold = debounceThreshold;
Vector3 center = touchable.transform.position;
float margin = 0.001f;
Vector3 pStart = center + new Vector3(0, 0, -touchableDistance - 0.5f);
Vector3 pTouch = center + new Vector3(0, 0, -touchableDistance + margin);
Vector3 pPoke = center + new Vector3(0, 0, -colliderThickness + margin);
Vector3 pDebounce = center + new Vector3(0, 0, -colliderThickness - touchable.DebounceThreshold - margin);
Vector3 pEnd = center + new Vector3(0, 0, touchableDistance + 0.5f);
// Test return beyond DebounceThreshold
yield return(PlayModeTestUtilities.MoveHand(pStart, pStart, gesture, handedness, inputSim, 1));
Assert.IsNull(pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pStart, pTouch, gesture, handedness, inputSim));
Assert.AreEqual(touchable, pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pTouch, pPoke, gesture, handedness, inputSim));
Assert.AreEqual(touchable, pokePointer.ClosestProximityTouchable);
Assert.AreEqual(objectDownExpected, pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pPoke, pDebounce, gesture, handedness, inputSim));
Assert.AreEqual(touchable, pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pDebounce, pStart, gesture, handedness, inputSim));
Assert.IsNull(pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
// Test touchable distance behind the surface
yield return(PlayModeTestUtilities.MoveHand(pStart, pStart, gesture, handedness, inputSim, 1));
Assert.IsNull(pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pStart, pTouch, gesture, handedness, inputSim));
Assert.AreEqual(touchable, pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pTouch, pPoke, gesture, handedness, inputSim));
Assert.AreEqual(touchable, pokePointer.ClosestProximityTouchable);
Assert.AreEqual(objectDownExpected, pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pPoke, pEnd, gesture, handedness, inputSim));
Assert.IsNull(pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pEnd, pDebounce, gesture, handedness, inputSim));
Assert.AreEqual(touchable, pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.MoveHand(pDebounce, pStart, gesture, handedness, inputSim));
Assert.IsNull(pokePointer.ClosestProximityTouchable);
Assert.IsNull(pokePointer.CurrentTouchableObjectDown);
yield return(PlayModeTestUtilities.HideHand(handedness, inputSim));
}
/// <inheritdoc />
public override void OnPreSceneQuery()
{
if (hand == null)
{
hand = Controller as IMixedRealityHand;
}
//
// Update query buffer which pointer position
//
Vector3 pointerPosition;
if (TryGetNearGraspPoint(out pointerPosition))
{
var layerMasks = PrioritizedLayerMasksOverride ?? GrabLayerMasks;
for (int i = 0; i < layerMasks.Length; i++)
{
if (queryBufferNearObjectRadius.TryUpdateQueryBufferForLayerMask(layerMasks[i], pointerPosition, triggerInteraction))
{
break;
}
}
for (int i = 0; i < layerMasks.Length; i++)
{
if (queryBufferInteractionRadius.TryUpdateQueryBufferForLayerMask(layerMasks[i], pointerPosition, triggerInteraction))
{
break;
}
}
}
//
// Update Rays with rays
//
// Create a rays cache, size base on how many rays will be created.
// Also there are more rays used when there is a focused item.
int focusedRayCount = 4;
int unfocusedRayCount = focusedRayCount - 1;
int currentRay = 0;
if (focusedRays == null || focusedRays.Length != focusedRayCount)
{
focusedRays = new RayStep[focusedRayCount];
}
if (unfocusedRays == null || unfocusedRays.Length != unfocusedRayCount)
{
unfocusedRays = new RayStep[unfocusedRayCount];
}
// Was the last result focusing a remote target?
bool hadRemoteFocus = RemoteResult != null && RemoteResult.IsTargetValid;
Rays = hadRemoteFocus ? focusedRays : unfocusedRays;
// First try creating web from thumb to other fingers. If that fails just use pointer position, and shoot rays along the axes.
float distance = SphereCastRadius + SphereCastRadius;
if (hand != null)
{
// Only query connection between the thumb and three fingers, so to reduce latency.
MixedRealityPose pinkyTip;
MixedRealityPose ringTip;
MixedRealityPose indexPose;
MixedRealityPose thumbPose;
if ((hand.TryGetJoint(TrackedHandJoint.ThumbTip, out thumbPose)) &&
(hand.TryGetJoint(TrackedHandJoint.RingTip, out ringTip)) &&
(hand.TryGetJoint(TrackedHandJoint.PinkyTip, out pinkyTip)) &&
(hand.TryGetJoint(TrackedHandJoint.IndexTip, out indexPose)))
{
// Could improve this if "RayStep" had an update function that took a position and direction.
var start = thumbPose.Position;
var end = start + ((indexPose.Position - start).normalized * distance);
Rays[currentRay++].UpdateRayStep(ref start, ref end);
end = start + ((ringTip.Position - start).normalized * distance);
Rays[currentRay++].UpdateRayStep(ref start, ref end);
end = start + ((pinkyTip.Position - start).normalized * distance);
Rays[currentRay++].UpdateRayStep(ref start, ref end);
}
}
else
{
Debug.LogFormat(LogType.Warning, LogOption.NoStacktrace, null, "The RemoteSphere pointer couldn't use hand joints. The results of this pointer may not be accurate.");
// Could improve this if "RayStep" had an update function that took a position and direction.
var end = pointerPosition + (Vector3.up * distance);
Rays[currentRay++].UpdateRayStep(ref pointerPosition, ref end);
end = pointerPosition + (Vector3.right * distance);
Rays[currentRay++].UpdateRayStep(ref pointerPosition, ref end);
end = pointerPosition + (Vector3.forward * distance);
Rays[currentRay++].UpdateRayStep(ref pointerPosition, ref end);
}
// If there was a previous target, try aimming for that target now
if (hadRemoteFocus)
{
// Could improve this if "RayStep" had an update function that took a position and direction.
var end = pointerPosition + ((RemoteResult.RemoteResult.HitPosition.toUnityPos() - pointerPosition).normalized * distance);
Rays[currentRay++].UpdateRayStep(ref pointerPosition, ref end);
}
}
void Update()
{
if (!photonView.IsMine)
{
if (_isShow)
{
if (((clientCounterFrames % clientUpdateFrameRate) == 0))
{
//now the wrist. lerp motion and rotation
GameObject wrist = GetJointObject(TrackedHandJoint.Wrist);
float lerpTime = Mathf.Clamp(InterpolatePointLerpSpeed * Time.smoothDeltaTime, 0.33f, 1.0f);
wrist.transform.localPosition = Vector3.Lerp(wrist.transform.localPosition, _rootPos, lerpTime);
Vector3 rotV = wrist.transform.localEulerAngles;
wrist.transform.localEulerAngles = Vector3.Lerp(rotV, _rootRot.Normalise(rotV - new Vector3(180, 180, 180), rotV + new Vector3(180, 180, 180)), lerpTime);
GameObject objectJoint;
float x, y, z, w;
for (int key = 0; key < _jointNameListArray.Length; key++)
{
int index = (int)_jointNameListArray[key];
if (index >= 0 && index < jointObjects.Length)
{
if (index == (int)TrackedHandJoint.Wrist)
{
continue;
}
objectJoint = jointObjects[index];
if (objectJoint != null)
{
int rIdx = (int)index * 4;
x = (float)ShortToFloat(_rotArrayShorts[rIdx + 0]);
y = (float)ShortToFloat(_rotArrayShorts[rIdx + 1]);
z = (float)ShortToFloat(_rotArrayShorts[rIdx + 2]);
w = (float)ShortToFloat(_rotArrayShorts[rIdx + 3]);
//lerp
if (x == 0 && y == 0 && z == 0 && w == 0)
{
//zero quaternions are invalid
//Debug.Log("Zero Quat: x==0&&y==0&&z==0&&w==0");
continue;
}
Quaternion rotQ = objectJoint.transform.localRotation;
objectJoint.transform.localRotation = Quaternion.Lerp(rotQ, new Quaternion(x, y, z, w).normalized, lerpTime);
}
}
}
clientCounterFrames = 0;
}
++clientCounterFrames;
}
}
else
{
bool isHandExist = HandJointUtils.TryGetJointPose(TrackedHandJoint.Wrist, Hand, out MixedRealityPose pose);
if (isHandExist)
{
if (_mixedRealityHand == null)
{
_mixedRealityHand = HandJointUtils.FindHand(Hand);
}
if (((localCounterFrames % localUpdateFrameRate) == 0))
{
SetJoint(TrackedHandJoint.Wrist, true);
// Enable the mesh if its disabled
if (!_isShow)
{
photonView.RPC("setMeshEnabled", RpcTarget.All, true);
}
// Set each of the joints
int count = _jointNameListArray.Length;
for (int i = 0; i < count; i++)
{
if (_jointNameListArray[i] != TrackedHandJoint.Wrist)
{
SetJoint(_jointNameListArray[i], false);
}
}
}
if ((localCounterFrames % clientSendFrameRate) == 0)
{
// Send the client local joints
GameObject wrist = GetJointObject(TrackedHandJoint.Wrist);
photonView.SendToAllUnReliableRPC(RPCRuntimeUPDShortcut, ReceiverGroup.Others, wrist.transform.localPosition, wrist.transform.localRotation, _rotArrayShorts);
localCounterFrames = 0;
}
}
else if (!isHandExist)
{
// Disable the mesh if the Hand Wrist doesn't exist
_mixedRealityHand = null;
if (_isShow)
{
photonView.RPC("setMeshEnabled", RpcTarget.All, false);
}
}
//always run the counter so its ready to send on first frame when the hand exists
++localCounterFrames;
}
}
public void OnSourceDetected(SourceStateEventData eventData)
{
hand = eventData.Controller as IMixedRealityHand;
}
