bool GetGesture(MLHand hand, MLHandKeyPose type)
{
if (hand != null)
{
if (hand.KeyPose == type)
{
if (hand.KeyPoseConfidence > 0.9f)
{
return(true);
}
}
}
return(false);
}
void Awake()
{
MLResult result = MLHands.Start();
if (!result.IsOk)
{
Debug.LogError("Error starting MLHand, disabling script.");
enabled = false;
return;
}
var enabledPoses = new MLHandKeyPose[] {
KEY_POSE
};
MLHands.KeyPoseManager.EnableKeyPoses(enabledPoses, true);
}
private void OnHandKeyPoseEnd(MLHandKeyPose gestureType, MLHandType handType)
{
if (gestureType == MLHandKeyPose.NoHand)
{
return;
}
if (handType == MLHandType.Left)
{
_leftRecognized = false;
}
else
{
_rightRecognized = false;
}
}
void Awake()
{

 var result = MLHands.Start(); 
 if (!result.IsOk)
{

 Debug.LogError("Error starting MLHands, disabling script."); 
 enabled = false; 
 return;
}


var enabledPoses = new MLHandKeyPose[] {
//MLHandKeyPose.C,
MLHandKeyPose.Finger,
MLHandKeyPose.Fist,
//MLHandKeyPose.L,
//MLHandKeyPose.Ok,
MLHandKeyPose.OpenHandBack,
//MLHandKeyPose.Pinch
//MLHandKeyPose.Thumb
};
MLHands.KeyPoseManager.EnableKeyPoses(enabledPoses, true);
}
// Update is called once per frame
private void Update()
{
MLHandKeyPose pose = MLHandKeyPose.NoHand;
if (MLHands.Right.KeyPoseConfidence >= RequiredConfidence && MLHands.Right.KeyPose != MLHandKeyPose.NoHand)
{
pose = MLHands.Right.KeyPose;
}
else if (MLHands.Left.KeyPoseConfidence >= RequiredConfidence)
{
pose = MLHands.Left.KeyPose;
}
switch (pose)
{
case MLHandKeyPose.C:
if (grabbed)
{
grabbed = false;
_body.velocity = Vector3.zero;
_body.angularVelocity = Vector3.zero;
_body.useGravity = true;
}
break;
case MLHandKeyPose.Pinch:
if (!grabbed)
{
grabbed = true;
_body.useGravity = false;
_body.velocity = Vector3.zero;
_body.angularVelocity = Vector3.zero;
}
break;
}
if (grabbed)
{
_body.transform.position = Camera.main.transform.position + Camera.main.transform.forward * GrabDistance;
}
}
void Awake()
{
MLResult hResult = MLHands.Start();
MLResult iResult = MLInput.Start();
MLResult pResult = MLWorldPlanes.Start();
MLResult wrResult = MLWorldRays.Start();
if (
!hResult.IsOk ||
!iResult.IsOk ||
!pResult.IsOk ||
!wrResult.IsOk
)
{
Debug.LogError("Error starting ML APIs, disabling script.");
enabled = false;
return;
}
// HANDS
var enabledPoses = new MLHandKeyPose[] {
MLHandKeyPose.Fist,
};
MLHands.KeyPoseManager.EnableKeyPoses(enabledPoses, true);
// INPUT
_iController = MLInput.GetController(MLInput.Hand.Left);
// WORLD PLANES
var queryParams = new MLWorldPlanesQueryParams();
var queryFlags = MLWorldPlanesQueryFlags.Horizontal | MLWorldPlanesQueryFlags.SemanticFloor;
queryParams.Flags = queryFlags;
queryParams.MaxResults = WP_MAX_PLANES;
MLWorldPlanes.GetPlanes(queryParams, HandleOnReceivedPlanes);
// WORLD RAYS
_wrBaseHeight = transform.position.y;
_wrLastCentimeters = (int)Math.Truncate(_wrBaseHeight * 100);
}
public bool ActionButtonDown(int button)
{
switch ((HandCommand)button)
{
case HandCommand.ThumbClick:
return(thumbClickState && thumbClickedToState);
}
return(false);
#if !XR_NON_MAGIC_LEAP
MLHandKeyPose pose = (MLHandKeyPose)button;
//Debug.Log("Look for action button down: " + pose);
if (mlHand.GetKeyPoseDown(pose))
{
gestureState[pose] = GestureDetectionFrameState.GestureBegan;
//Debug.Log("YESS action button down: " + pose);
return(true);
}
#endif // #if !XR_NON_MAGIC_LEAP
return(false);
}
public bool ActionButtonUp(int button)
{
switch ((HandCommand)button)
{
case HandCommand.ThumbClick:
return(!thumbClickState && thumbClickedToState);
}
return(false);
#if !XR_NON_MAGIC_LEAP
MLHandKeyPose pose = (MLHandKeyPose)button;
if (mlHand.GetKeyPoseUp(pose))
{
gestureState[pose] = GestureDetectionFrameState.GestureReleased;
return(true);
}
#endif
//if (OVRInput.GetUp((OVRInput.Button)button)) {
// return true;
//}
return(false);
}
public bool VerifyKeyPoseConfidence(MLHandKeyPose pose, bool isLeftHand, float confidenceThreshold = 0.3f)
{
if (isLeftHand)
{
if (leftKeyPose == pose)
{
if (leftPoseConfidence > (1.0f - confidenceThreshold))
{
return(true);
}
else
{
return(false);
}
}
else
{
return(false);
}
}
else
{
if (rightKeyPose == pose)
{
if (rightPoseConfidence > (1.0f - confidenceThreshold))
{
return(true);
}
else
{
return(false);
}
}
else
{
return(false);
}
}
}
// Update is called once per frame
void Update()
{
rightHandPose = handTrackingManager.GetRightKeyPose();
leftHandPose = handTrackingManager.GetLeftKeyPose();
switch (gameSequenceInt)
{
case 1:
{
if (!gesturesAnim.activeSelf)
{
gesturesAnim.SetActive(true);
UIInstructionText.text = "With your left hand extended, please make a fist then open your hand";
}
CheckHasSpawnedHuman();
}
break;
case 2:
{
if (!swipeAnim.activeSelf)
{
swipeAnim.SetActive(true);
UIInstructionText.text = "Please swipe on the touchpad of your Magic Leap controller";
}
CheckHasFocusBrain();
}
break;
case 3:
{
CheckHasTextHighlight();
};
break;
}
}
private void HandleGestureChanged(ManagedHand hand, MLHandKeyPose pose)
{
//snap transform changes to avoid force pops:
switch (pose)
{
case MLHandKeyPose.Fist:
_handCollider.transform.position = _managedHand.Skeleton.Position;
_handCollider.transform.localScale = Vector3.one * _managedHand.Gesture.Grasp.radius;
break;
case MLHandKeyPose.L:
case MLHandKeyPose.Finger:
_handCollider.transform.localScale = Vector3.one * _pointColliderSize;
_handCollider.transform.position = _managedHand.Skeleton.Index.Tip.positionFiltered;
break;
case MLHandKeyPose.OpenHand:
_handCollider.transform.localScale = new Vector3(_managedHand.Gesture.Grasp.radius, _openHandColliderWidth, _managedHand.Gesture.Grasp.radius);
_handCollider.transform.position = _managedHand.Skeleton.Position;
_handCollider.transform.rotation = _managedHand.Skeleton.Rotation;
break;
}
}
// Update is called once per frame
void Update()
{
var hPosition = Hand.Index.Tip.Position;
var rayOrigin = Hand.Wrist.Center.Position;
var rayDirection = Hand.Index.PIP.Position;
if (MLHands.IsStarted)
{
MLHandKeyPose pose = KeyPose(Hand);
if (pose == MLHandKeyPose.Pinch)
{
//obj.SetActive(true);
obj.transform.position = hPosition;
obj.GetComponent <ActionStates>().deactivateSpider();
}
if (Hand.GetKeyPoseDown(MLHandKeyPose.OpenHandBack))
{
obj.GetComponent <ActionStates>().backUp();
}
if (pose == MLHandKeyPose.Finger)
{
if (Hand.GetKeyPoseDown(MLHandKeyPose.Finger))
{
lPosition = rayDirection;
lDirection = rayDirection - rayOrigin;
}
else
{
lPosition = Vector3.MoveTowards(lPosition, rayDirection, speed * Time.deltaTime);
lDirection = Vector3.MoveTowards(lDirection, rayDirection - rayOrigin, speed * Time.deltaTime);
}
CastRay(lPosition, lDirection);
}
}
}
void Hand_OnKeyPoseBegin(MLHandKeyPose obj)
{
if (MLHands.IsStarted && mlHand.HandConfidence < 0.85f)
{
return;
}
//Debug.Log(name + ": KEY POSE BEGIN: " + obj);
SetHandObjectsActive(true);
MLHandKeyPose lastPose = currentPose;
currentPose = obj;
if (currentPose != MLHandKeyPose.NoHand &&
(!MLHands.IsStarted || mlHand.HandConfidence > 0.85f))
{
SetHandObjectsActive(true);
}
if (textPoseInfo != null)
{
textPoseInfo.text = "POSE: " + currentPose.ToString();
Debug.Log(textPoseInfo.text);
}
float poseTime = Time.time - lastPoseTime;
//Debug.Log("Check for onhandposebegin: " + OnHandPoseBegin);
if (OnHandPoseBegin != null)
{
//Debug.Log("On hand pose begin: " + obj);
OnHandPoseBegin(this, obj);
}
lastPoseTime = Time.time;
}
// Update is called once per frame
void Update()
{
var camera = Camera.main;
var speed = 0.1f;
float step = speed * Time.deltaTime; // calculate distance to move
var moveVector = transform.position - camera.transform.position;
var forward2d = Vector3.forward;
forward2d.y = 0.0f;
var camera2d = camera.transform.position;
camera2d.y = 0.0f;
if (MLHands.IsStarted)
{
MLHandKeyPose pose = KeyPose(MLHands.Left);
if (pose == MLHandKeyPose.OpenHandBack)
{
transform.Translate(forward2d.x * Time.deltaTime, 0, forward2d.z * Time.deltaTime, Camera.main.transform);
}
}
}
// Update is called once per frame
void Update()
{
if (MLHands.IsStarted)
{
leftKeyPose = MLHands.Left.KeyPose;
rightKeyPose = MLHands.Right.KeyPose;
leftHandPosition = MLHands.Left.Center;
rightHandPosition = MLHands.Right.Center;
leftPoseConfidence = MLHands.Left.KeyPoseConfidence;
rightPoseConfidence = MLHands.Right.KeyPoseConfidence;
lastFrameHandOffsetHint = thisFrameHandOffsetHint;
thisFrameHandOffsetHint = leftHandPosition - rightHandPosition;
lastFrameHandOffset = thisFrameHandOffset;
thisFrameHandOffset = rightHandPosition - leftHandPosition;
}
else
{
Debug.LogError("The Magic Leap hand tracking was unable to initialize");
}
}
//Event Handlers:
private void HandleKeyposeChanged(MLHandKeyPose keyPose)
{
OnKeyPoseChanged?.Invoke(_managedHand, keyPose);
}
//Public Methods:
public void Update()
{
if (!_managedHand.Visible)
{
return;
}
//pinch rotation offset mirror:
Vector3 rotationOffset = _pinchAbsoluteRotationOffset;
if (_managedHand.Hand.HandType == MLHandType.Left)
{
rotationOffset.y *= -1;
}
//holders:
Vector3 pinchPosition = Vector3.zero;
Quaternion pinchRotation = Quaternion.identity;
//pinch interaction point radius:
if (_managedHand.Skeleton.Thumb.Tip.Visible && _managedHand.Skeleton.Index.Tip.Visible)
{
Pinch.radius = Vector3.Distance(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered);
}
if (_managedHand.Skeleton.Thumb.Tip.Visible) //absolute placement:
{
//are we swapping modes?
if (_pinchIsRelative)
{
_pinchIsRelative = false;
_pinchTransitioning = true;
_pinchTransitionStartTime = Time.realtimeSinceStartup;
}
pinchPosition = _managedHand.Skeleton.Thumb.Tip.positionFiltered;
pinchRotation = TransformUtilities.RotateQuaternion(_managedHand.Skeleton.Rotation, rotationOffset);
//gather offset distance:
if (_managedHand.Skeleton.Index.Knuckle.Visible && _managedHand.Skeleton.Thumb.Knuckle.Visible)
{
Vector3 mcpMidpoint = Vector3.Lerp(_managedHand.Skeleton.Index.Knuckle.positionFiltered, _managedHand.Skeleton.Thumb.Knuckle.positionFiltered, .5f);
_pinchRelativePositionDistance = Vector3.Distance(mcpMidpoint, pinchPosition);
}
}
else //relative placement:
{
//are we swapping modes?
if (!_pinchIsRelative)
{
_pinchIsRelative = true;
_pinchTransitioning = true;
_pinchTransitionStartTime = Time.realtimeSinceStartup;
}
//place between available mcps:
if (_managedHand.Skeleton.Index.Knuckle.Visible && _managedHand.Skeleton.Thumb.Knuckle.Visible)
{
pinchPosition = Vector3.Lerp(_managedHand.Skeleton.Index.Knuckle.positionFiltered, _managedHand.Skeleton.Thumb.Knuckle.positionFiltered, .5f);
//rotate:
pinchRotation = TransformUtilities.RotateQuaternion(_managedHand.Skeleton.Rotation, _pinchRelativeRotationOffset);
//move out along rotation forward:
pinchPosition += pinchRotation * Vector3.forward * _pinchRelativePositionDistance;
}
else
{
//just use previous:
pinchPosition = Pinch.position;
pinchRotation = Pinch.rotation;
}
}
//sticky release reduction:
if (_collapsed)
{
if (_managedHand.Skeleton.Thumb.Tip.Visible && _managedHand.Skeleton.Index.Tip.Visible)
{
//if starting to release, start using a point between the thumb and index tips:
if (Vector3.Distance(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered) > _dynamicReleaseDistance)
{
pinchPosition = Vector3.Lerp(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered, .3f);
}
}
}
//apply pinch pose - to avoid jumps when relative placement is used we smooth until close enough:
if (_pinchTransitioning)
{
//position:
Pinch.position = Vector3.SmoothDamp(Pinch.position, pinchPosition, ref _pinchArrivalPositionVelocity, _pinchTransitionTime);
float positionDelta = Vector3.Distance(Pinch.position, pinchPosition);
//rotation:
Pinch.rotation = MotionUtilities.SmoothDamp(Pinch.rotation, pinchRotation, ref _pinchArrivalRotationVelocity, _pinchTransitionTime);
float rotationDelta = Quaternion.Angle(Pinch.rotation, pinchRotation);
//close enough to hand off?
if (positionDelta < .001f && rotationDelta < 5)
{
_pinchTransitioning = false;
}
//taking too long?
if (Time.realtimeSinceStartup - _pinchTransitionStartTime > _pinchTransitionMaxDuration)
{
_pinchTransitioning = false;
}
}
else
{
Pinch.position = pinchPosition;
Pinch.rotation = pinchRotation;
}
//grasp interaction point:
Bounds graspBounds = CalculateGraspBounds
(
_managedHand.Skeleton.Thumb.Knuckle,
_managedHand.Skeleton.Thumb.Joint,
_managedHand.Skeleton.Thumb.Tip,
_managedHand.Skeleton.Index.Knuckle,
_managedHand.Skeleton.Index.Joint,
_managedHand.Skeleton.Index.Tip,
_managedHand.Skeleton.Middle.Knuckle,
_managedHand.Skeleton.Middle.Joint,
_managedHand.Skeleton.Middle.Tip
);
Grasp.position = _managedHand.Skeleton.Position;
//when points are being initially found they can be wildly off and this could cause a massively large volume:
Grasp.radius = Mathf.Min(graspBounds.size.magnitude, _maxGraspRadius);
Grasp.rotation = _managedHand.Skeleton.Rotation;
//intent updated:
if (_currentInteractionState != null)
{
_currentInteractionState.FireUpdate();
}
//keypose change proposed:
if (_managedHand.Hand.KeyPose != VerifiedGesture && _managedHand.Hand.KeyPose != _proposedKeyPose)
{
//queue a new proposed change to keypose:
_proposedKeyPose = _managedHand.Hand.KeyPose;
_keyPoseChangedTime = Time.realtimeSinceStartup;
}
//keypose change acceptance:
if (_managedHand.Hand.KeyPose != VerifiedGesture && Time.realtimeSinceStartup - _keyPoseChangedTime > _keyPoseStabailityDuration)
{
//reset:
Point.active = false;
Pinch.active = false;
Grasp.active = false;
if (_collapsed)
{
//intent end:
if (_managedHand.Hand.KeyPose == MLHandKeyPose.C || _managedHand.Hand.KeyPose == MLHandKeyPose.OpenHand || _managedHand.Hand.KeyPose == MLHandKeyPose.L || _managedHand.Hand.KeyPose == MLHandKeyPose.Finger)
{
if (_managedHand.Skeleton.Thumb.Tip.Visible && _managedHand.Skeleton.Index.Tip.Visible)
{
//dynamic release:
if (Vector3.Distance(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered) > _dynamicReleaseDistance)
{
//end intent:
_collapsed = false;
_currentInteractionState.FireEnd();
_currentInteractionState = null;
//accept keypose change:
VerifiedGesture = _managedHand.Hand.KeyPose;
_proposedKeyPose = _managedHand.Hand.KeyPose;
OnVerifiedGestureChanged?.Invoke(_managedHand, VerifiedGesture);
if (_managedHand.Hand.KeyPose == MLHandKeyPose.Finger || _managedHand.Hand.KeyPose == MLHandKeyPose.L)
{
Intent = IntentPose.Pointing;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
else if (_managedHand.Hand.KeyPose == MLHandKeyPose.C || _managedHand.Hand.KeyPose == MLHandKeyPose.OpenHand || _managedHand.Hand.KeyPose == MLHandKeyPose.Thumb)
{
Intent = IntentPose.Relaxed;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
}
}
}
}
else
{
//intent begin:
if (_managedHand.Hand.KeyPose == MLHandKeyPose.Pinch || _managedHand.Hand.KeyPose == MLHandKeyPose.Ok || _managedHand.Hand.KeyPose == MLHandKeyPose.Fist)
{
_collapsed = true;
if (_managedHand.Hand.KeyPose == MLHandKeyPose.Pinch || _managedHand.Hand.KeyPose == MLHandKeyPose.Ok)
{
Intent = IntentPose.Pinching;
Pinch.active = true;
_currentInteractionState = Pinch.Touch;
_currentInteractionState.FireBegin();
OnIntentChanged?.Invoke(_managedHand, Intent);
}
else if (_managedHand.Hand.KeyPose == MLHandKeyPose.Fist)
{
Intent = IntentPose.Grasping;
Grasp.active = true;
_currentInteractionState = Grasp.Touch;
_currentInteractionState.FireBegin();
OnIntentChanged?.Invoke(_managedHand, Intent);
}
}
if (_managedHand.Hand.KeyPose == MLHandKeyPose.Finger || _managedHand.Hand.KeyPose == MLHandKeyPose.L)
{
Intent = IntentPose.Pointing;
Point.active = true;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
else if (_managedHand.Hand.KeyPose == MLHandKeyPose.C || _managedHand.Hand.KeyPose == MLHandKeyPose.OpenHand || _managedHand.Hand.KeyPose == MLHandKeyPose.Thumb)
{
Intent = IntentPose.Relaxed;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
//accept keypose change:
VerifiedGesture = _managedHand.Hand.KeyPose;
_proposedKeyPose = _managedHand.Hand.KeyPose;
OnVerifiedGestureChanged?.Invoke(_managedHand, VerifiedGesture);
}
}
}
public void BeginKeyPose(MLHandKeyPose pose)
{
}
bool DoesGestureMatch(MLHand hand, MLHandKeyPose type)
{
return(hand != null && hand.KeyPose == type && hand.KeyPoseConfidence > 0.9f);
}
bool CheckForGesture(MLHandKeyPose type)
{
return(DoesGestureMatch(MLHands.Left, type) || DoesGestureMatch(MLHands.Right, type));
}
public void EndKeyPose(MLHandKeyPose pose)
{
}
public static bool IsOpenHand(this MLHandKeyPose keyPose) => keyPose.ToString() == "OpenHandBack";
//private Vector3 AverageMovement(Vector3[] arr)
//{
// var av = Vector3.zero;
// foreach (Vector3 vec in arr)
// av += vec;
// av /= arr.Length;
// Debug.Log(av);
// return av;
//}
private void DragonPoseLogic()
{
if (MLHands.Left.KeyPose == MLHandKeyPose.Pinch && lastPoseLeft != MLHandKeyPose.Pinch)
{
//startPos = AverageMovement(posiStackLeft);
startPos = MLHands.Left.Center;
timer = 0;
}
if (MLHands.Right.KeyPose == MLHandKeyPose.Pinch && lastPoseRight != MLHandKeyPose.Pinch)
{
//startPos = AverageMovement(posiStackRight);
startPos = MLHands.Right.Center;
timer = 0;
}
if (MLHands.Left.KeyPose == MLHandKeyPose.Pinch || MLHands.Right.KeyPose == MLHandKeyPose.Pinch)
{
if (0 == timer)
{
if (MLHands.Left.KeyPose == MLHandKeyPose.Pinch)
{
//startPos = AverageMovement(posiStackLeft);
startPos = MLHands.Left.Center;
}
else if (MLHands.Right.KeyPose == MLHandKeyPose.Pinch)
{
//startPos = AverageMovement(posiStackRight);
startPos = MLHands.Right.Center;
}
}
timer += Time.deltaTime;
//Debug.Log("1");
//move
if (Mathf.Abs(startPos.y - transform.position.y) >= travelDistance)
{
RaycastHit hit;
Physics.Raycast(Camera.main.transform.position, Camera.main.transform.forward, out hit, 100f);
//Debug.Log("2");
if (null != hit.transform)
{
PlayerCharacter.Instance.MoveToTarget(hit.point);
}
}
if (timer >= travelTime)
{
timer = 0;
}
}
else
{
timer = 0;
if (MLHands.Left.KeyPose == MLHandKeyPose.C && lastPoseLeft != MLHandKeyPose.C ||
MLHands.Right.KeyPose == MLHandKeyPose.C && lastPoseRight != MLHandKeyPose.C)
{
Debug.Log("fireball");
}
}
if (MLHands.Left.KeyPose != lastPoseLeft)
{
lastPoseLeft = MLHands.Left.KeyPose;
}
if (MLHands.Right.KeyPose != lastPoseRight)
{
lastPoseRight = MLHands.Right.KeyPose;
}
}
//----------- Public Methods -----------
public static bool Matches(MLHand hand, MLHandKeyPose keyPose, MLHandType handType)
{
return(hand.KeyPose == keyPose && hand.HandType == handType);
}
private void OnGestureEnd(MLHandType handType, MLHandKeyPose keyPose)
{
MLHand hand = handType == MLHandType.Left ? MLHands.Left : MLHands.Right;
EnableContent(hand, false);
}
public static bool IsFist(this MLHandKeyPose keyPose) => keyPose.ToString() == "Fist";
/// <summary>
/// Convenience function, returns true of both hands are contained in the given gestureTypes bitmask.
/// </summary>
public static bool Matches(MLHandKeyPose leftPose, MLHandKeyPose rightPose, KeyPoseTypes keyPose)
{
return(Matches(leftPose, keyPose) && Matches(rightPose, keyPose));
}
// We subscribe to the event "OnKeyPoseBegin" which basically fires right when a gesture is being detected.
//private void OnEnable()
//{
// MLHands.Right.OnKeyPoseBegin += HandPoseReaction;
// MLHands.Left.OnKeyPoseBegin += HandPoseReaction;
//}
// Normally you'd put here your reactive code; what would happen when a specific gesture is recognized. Here we simply just print the gesture's name on screen.
private void HandPoseReaction(MLHandKeyPose _pose)
{
Debug.Log(_pose.ToString());
}
