//to be override as detecting if the gesture is still in process
public override bool checkInProcess()
{
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
//Debug.Log("(Hovering)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
_tipMovement = UnityVectorExtension.ToVector3(finger.TipPosition) - _lastTipPos;
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}
}
_lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
Debug.Log("(Hovering)Gesture InProcess");
return(true);
}
//to be override as detecting if the gesture is still in process
public override bool checkInProcess()
{
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
//Debug.Log("(Grab)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
_palmMovement = UnityVectorExtension.ToVector3(hand.PalmPosition) - _lastPalmPos;
_lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
Debug.Log("(Grabbing) Gesture InProcess");
return(true);
}
void updateBall()
{
bool isUpdating = false;
if (_isHoldingBall)
{
Hand hand = GetCurrent1Hand();
if (hand != null)
{
if (isGrabHand(hand))
{
if (CurrentHoldingObj != null)
{
CurrentHoldingObj.transform.position = UnityVectorExtension.ToVector3(hand.PalmPosition + hand.PalmNormal.Normalized * 0.03f);
isUpdating = true;
}
}
}
if (!isUpdating)
{
releaseBall();
}
}
}
protected Vector3 getHandVelocity(Hand hand)
{
if (player == null)
{
player = transform.root;
}
return(player.InverseTransformDirection(UnityVectorExtension.ToVector3(hand.PalmVelocity)));
}
// Update is called once per frame
void Update()
{
Frame frame = controller.Frame();
List <Hand> hands = frame.Hands;
int f_cnt = hands.Count * 5;
int f_hidden_cnt = 2 * 5 - f_cnt;
int h_cnt = 2;
int ff_cnt = 0;
for (int idxFinger = 0; idxFinger < f_hidden_cnt; ++idxFinger)
{
var fingerObj = FingerObjects[ff_cnt++];
fingerObj.SetActive(false);
}
for (int i = 0; i < h_cnt; i++)
{
List <Finger> fingers = hands[i].Fingers;
for (int idxFinger = 0; idxFinger < fingers.Count; ++idxFinger)
{
Finger finger = fingers[idxFinger];
Vector fingerPos = finger.TipPosition;
Color set_collor = new Color(0, 0, 0);
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
set_collor = new Color(255, 0, 0);
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
set_collor = new Color(0, 255, 0);
}
else if (finger.Type == Finger.FingerType.TYPE_MIDDLE)
{
set_collor = new Color(0, 0, 255);
}
else if (finger.Type == Finger.FingerType.TYPE_RING)
{
set_collor = new Color(255, 255, 0);
}
else if (finger.Type == Finger.FingerType.TYPE_PINKY)
{
set_collor = new Color(0, 255, 255);
}
else
{
set_collor = new Color(255, 255, 255);
}
var fingerObj = FingerObjects[ff_cnt++];
fingerObj.transform.position = UnityVectorExtension.ToVector3(fingerPos) / 10;
fingerObj.GetComponent <Renderer>().material.color = set_collor;
fingerObj.SetActive(true);
}
}
}
//to be override as detecting the start signal of a gusture
public override bool checkStart()
{
//1. Only one hand input
//2. Thumb far from Index
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
Debug.Log("(Pinch)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = finger.TipPosition.ToVector3();
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
indexTip = finger.TipPosition.ToVector3();
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}
}
float lastTipGap = Vector3.Distance(thumbTip, indexTip);
if (lastTipGap > 0.8 * hand.PalmWidth)
{
Debug.Log("(Pinch)Gesture Start");
return(true);
}
else
{
Debug.Log("(Pinch)Gesture Start fingers distance problem");
return(false);
}
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
//to be override as detecting if the gesture is still in process
public override bool checkInProcess()
{
//1. Only one hand input
//2. The fingers are moving closer
//3. Time is in limit
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = finger.TipPosition.ToVector3();
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
indexTip = finger.TipPosition.ToVector3();
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}
}
if (lastTipGap >= Vector3.Distance(thumbTip, indexTip))
{
lastTipGap = Vector3.Distance(thumbTip, indexTip);
Debug.Log("(Pinch)Gesture InProcess");
return(true);
}
else
{
lastTipGap = Vector3.Distance(thumbTip, indexTip);
return(false);
}
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
//to be override as detecting the start signal of a gusture
public override bool checkStart()
{
//1. Only one hand input
//2. The fingers have moved around hand
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
//Debug.Log("(Grab)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = finger.TipPosition.ToVector3();
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
indexTip = finger.TipPosition.ToVector3();
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}
}
_lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
if (Vector3.Distance(thumbTip, indexTip) < 0.3 * hand.PalmWidth)
{
Debug.Log("(Pinching)Gesture start");
return(true);
}
else
{
Debug.Log("(Pinching)Gesture start problem");
return(false);
}
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
//to be override as detecting if the gesture is finished
public override bool checkEnd()
{
//1. Only one hand input
//2. The direction of fingers (no thumb) are close to the palm direction
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = finger.TipPosition.ToVector3();
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
_tipMovement = UnityVectorExtension.ToVector3(finger.TipPosition) - _lastTipPos;
indexTip = finger.TipPosition.ToVector3();
}
}
_lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
if (Vector3.Distance(thumbTip, indexTip) > 0.3 * hand.PalmWidth)
{
Debug.Log("(Pinching)Gesture end");
return(true);
}
else
{
//Debug.Log("(Pinching)Gesture end problem");
return(false);
}
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
//to be override as detecting if the gesture is finished
public override bool checkEnd()
{
//1. Only one hand input
//2. The fingers have moved together
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = finger.TipPosition.ToVector3();
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
indexTip = finger.TipPosition.ToVector3();
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}
}
lastTipGap = Vector3.Distance(thumbTip, indexTip);
if (lastTipGap < 0.3 * hand.PalmWidth)
{
return(true);
}
else
{
Debug.Log("Pinch End Problem");
return(false);
}
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
//to be override as detecting the start signal of a gusture
public override bool checkStart()
{
//1. Only one hand input
//2. The direction of fingers (no thumb) are close to the palm direction
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
//Debug.Log("(Grab)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
currentGrabAngle = hand.GrabAngle;
_lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
return(currentGrabAngle < 2);
/*
* currentFingerAngle.Clear();
* foreach (Finger finger in hand.Fingers)
* {
* if (finger.Type != Finger.FingerType.TYPE_THUMB && finger.Type != Finger.FingerType.TYPE_PINKY)
* {
* float angle = Vector3.Angle(hand.Direction.ToVector3(), finger.Direction.ToVector3());
* currentFingerAngle.Add(finger.Type, angle);
* if ( angle > 90.0f)
* {
* Debug.Log("(Grab)Start finger direction problem ");
* return false;
* }
* }
* }
* Debug.Log("(Grab)Gesture start");
* _lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
* return true;*/
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
void throwBall()
{
Hand hand = GetCurrent1Hand();
if (hand != null)
{
GameObject go = GameObject.Instantiate(prefabBall);
go.transform.position = UnityVectorExtension.ToVector3(hand.PalmPosition);
setupGravity(go);
addForce(go, UnityVectorExtension.ToVector3(hand.PalmVelocity * forceToAdd));
}
}
protected override bool checkConditionGesture()
{
Hand hand = GetCurrent1Hand();
if (hand != null)
{
if (isPalmNormalSameDirectionWith(hand, UnityVectorExtension.ToVector3(hand.PalmVelocity)) &&
!isStationary(hand))
{
return(true);
}
}
return(false);
}
void updateDebug()
{
if (_listHands != null && _listHands.Count > 0 && _listHands.Count <= 2)
{
foreach (Hand hand in _listHands)
{
Debug.DrawRay(UnityVectorExtension.ToVector3(hand.PalmPosition), UnityVectorExtension.ToVector3(hand.PalmNormal) * 10, Color.green);
if (!_gestureManager.isStationary(hand, smallestVelocity))
{
Debug.DrawRay(UnityVectorExtension.ToVector3(hand.PalmPosition), UnityVectorExtension.ToVector3(hand.PalmVelocity) * 10, Color.blue);
}
}
}
}
protected override bool checkConditionGesture()
{
Hand hand = GetCurrent1Hand();
if (hand != null)
{
if (_gestureManager.isPalmNormalSameDirectionWith(hand, UnityVectorExtension.ToVector3(hand.PalmVelocity), handForwardDegree) &&
!_gestureManager.isStationary(hand, smallestVelocity))
{
return(true);
}
}
return(false);
}
void updateDebug()
{
if (listHands.Count > 0 && listHands.Count <= 2)
{
foreach (Hand hand in listHands)
{
Debug.DrawRay(UnityVectorExtension.ToVector3(hand.PalmPosition), UnityVectorExtension.ToVector3(hand.PalmNormal) * 10, Color.green);
if (!isStationary(hand))
{
Debug.DrawRay(UnityVectorExtension.ToVector3(hand.PalmPosition), UnityVectorExtension.ToVector3(hand.PalmVelocity) * 10, Color.blue);
}
}
}
}
//to be override as detecting the start signal of a gusture
public override bool checkStart()
{
//1. Only one hand input
//2. The fingers have moved around hand
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
//Debug.Log("(Grab)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
_lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
_palmMovement = new Vector3();
return(hand.GrabAngle > 3); /*
* foreach (Finger finger in hand.Fingers)
* {
* if (finger.Type != Finger.FingerType.TYPE_THUMB && finger.Type != Finger.FingerType.TYPE_PINKY)
* {
* float angle = Vector3.Angle(hand.Direction.ToVector3(), finger.Direction.ToVector3());
* if ( angle < 120.0f)
* {
* Debug.Log("(Grabbing)Start finger direction problem ");
* return false;
* }
* }
* }
* Debug.Log("(Grabbing)Gesture start");
* _lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
* return true;*/
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
protected void setText(Hand hand, GameObject handInfo)
{
if (handInfo == null)
{
return;
}
string handInfo_str = "";
/*
* foreach (Finger finger in hand.Fingers)
* {
* //handInfo_str += (finger.Type + ":" + Vector3.Angle(finger.Direction.ToVector3(), hand.Direction.ToVector3()) + "\n");
* handInfo_str += (finger.Type + ":" + finger.TipPosition.ToVector3() + "\n");
* }
*/
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
//handInfo_str += ("\n" + finger.Type + ":" + finger.IsExtended);
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = UnityVectorExtension.ToVector3(finger.TipPosition);
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
indexTip = UnityVectorExtension.ToVector3(finger.TipPosition);
}
}
//handInfo_str += ("PalmarAxis = " + hand.PalmarAxis());
handInfo_str += ("\nPalmWidh = " + hand.PalmWidth);
handInfo_str += ("\nGrabAngle = " + hand.GrabAngle);
//handInfo_str += ("\nGrabStrength = " + hand.GrabStrength);
//handInfo_str += ("\nPalmPosition = " + UnityVectorExtension.ToVector3(hand.PalmPosition));
//handInfo_str += ("\nPalmVelocity = " + hand.PalmVelocity.ToVector3());
handInfo_str += ("\nTipGap = " + Vector3.Distance(thumbTip, indexTip));
handInfo_str += ("\nRate = " + Vector3.Distance(thumbTip, indexTip) / hand.PalmWidth);
try
{
handInfo.GetComponent <Text>().text = handInfo_str;
}
catch
{
handInfo.GetComponent <TextMesh>().text = handInfo_str;
}
}
//to be override as detecting the start signal of a gusture
public override bool checkStart()
{
//1. Only one hand input
//2. The fingers have moved around hand
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
//Debug.Log("(Hovering)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
indexTip = finger.TipPosition.ToVector3();
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}
if (!finger.IsExtended)
{
return(false);
}
}
_lastPalmPos = UnityVectorExtension.ToVector3(hand.PalmPosition);
return(true);
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
void Update()
{
controller = new Controller();
Frame frame = controller.Frame();
List <Hand> hands = frame.Hands;
if (frame.Hands.Count > 0)
{
firstHand = hands[0];
thumbL = firstHand.Fingers[0];
pinkyL = firstHand.Fingers[4];
if (thumbL.IsExtended && pinkyL.IsExtended && Time.time > nextFire)
{
nextFire = Time.time + fireRate; // Prevents spamming shots
Instantiate(shot, shotSpawn.position, shotSpawn.rotation);
GetComponent <AudioSource>().Play(); // Adds audio when firing
}
Vector3 palmNormal = UnityVectorExtension.ToVector3(firstHand.PalmNormal); //using palm rotation
Vector3 movement = new Vector3(-palmNormal[0], 0.0f, palmNormal[2]);
//Vector3 palmPos = UnityVectorExtension.ToVector3(firstHand.PalmPosition); //using hand position
//Vector3 movement = new Vector3(palmPos[0] * 0.01f , 0.0f, -palmPos[2] * 0.01f);
GetComponent <Rigidbody>().velocity = movement * speed;
}
GetComponent <Rigidbody>().position = new Vector3
(
Mathf.Clamp(GetComponent <Rigidbody>().position.x, boundary.xMin, boundary.xMax),
0.0f,
Mathf.Clamp(GetComponent <Rigidbody>().position.z, boundary.zMin, boundary.zMax)
);
GetComponent <Rigidbody>().rotation = Quaternion.Euler(0.0f, 0.0f, GetComponent <Rigidbody>().velocity.x * -tilt);
}
private void generateDataForGesture(Controller controller)
{
string fingerPositionString = "";
string fingerDistanceString = "";
int numberOfFingers;
string currentHand = "";
Frame frame = controller.Frame();
numberOfFingers = 0;
List <Hand> hands = frame.Hands;
for (int h = 0; h < frame.Hands.Count; h++)
{
Hand leapHand = frame.Hands[h];
Vector handXBasis = leapHand.PalmNormal.Cross(leapHand.Direction).Normalized;
Vector handYBasis = -leapHand.PalmNormal;
Vector handZBasis = -leapHand.Direction;
Vector handOrigin = leapHand.PalmPosition;
Matrix handTransform = new Matrix(handXBasis, handYBasis, handZBasis, handOrigin);
handTransform = handTransform.RigidInverse();
numberOfFingers += hands[h].Fingers.Count;
if (leapHand.IsLeft)
{
currentHand = "100";
}
if (leapHand.IsRight)
{
currentHand = "101";
}
fingerPositionString += currentHand + ",";
if (leapHand.Fingers.Count > 0)
{
for (int f = 0; f < leapHand.Fingers.Count; f++)
{
Finger leapFinger = leapHand.Fingers[f];
Vector transformedPosition = handTransform.TransformPoint(leapFinger.TipPosition);
Vector transformedDirection = handTransform.TransformDirection(leapFinger.Direction);
float fingerLength = leapFinger.Length;
Finger nextLeapFinger = null;
Vector nextFingerPosition = new Vector();
if (f == 0)
{
nextLeapFinger = leapHand.Fingers[1];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 1)
{
nextLeapFinger = leapHand.Fingers[2];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 2)
{
nextLeapFinger = leapHand.Fingers[3];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 3)
{
nextLeapFinger = leapHand.Fingers[4];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 4)
{
nextLeapFinger = leapHand.Fingers[0];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
float Dist = DistanceBetweenTwoPoints(UnityVectorExtension.ToVector3(transformedPosition), UnityVectorExtension.ToVector3(nextFingerPosition));
fingerPositionString += transformedPosition.x + "," + transformedPosition.y + "," + transformedPosition.z + ",";
fingerDistanceString += (int)Dist + ",";
}
}
}
fingerPositionString += fingerDistanceString;
fingerDistanceString = "";
gestureData = (numberOfFingers + "," + fingerPositionString).TrimEnd(',');
fingerPositionString = "";
}
//to be override as detecting if the gesture is finished
public override bool checkEnd()
{
//1. Only one hand input
//2. The direction of fingers (no thumb) are close to the palm direction
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
bool tappingEnd = false;
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = finger.TipPosition.ToVector3();
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
float angle = Vector3.Angle(hand.Direction.ToVector3(), finger.Direction.ToVector3());
if (angle > 60.0f)
{
Debug.Log("(Tapping)End index finger direction problem ");
tappingEnd = true;
}
indexTip = finger.TipPosition.ToVector3();
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}/*
* else
* {
* float angle = Vector3.Angle(hand.Direction.ToVector3(), finger.Direction.ToVector3());
* if (angle < 120.0f)
* {
* Debug.Log("(Tapping)End other fingers direction problem ");
* tappingEnd = true;
* }
* }*/
}
if (Vector3.Distance(thumbTip, indexTip) < 0.3 * hand.PalmWidth)
{
Debug.Log("(Tapping)Gesture End problem");
tappingEnd = true;
}
return(tappingEnd);
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
protected void onGraspEndActived()
{
_isLocked = true;
_isGrasped = false;
cancelHighLightText();
if (_isLongGrasped)
{
_isLongGrasped = false;
if (!LayerManager.handle.updateLayerExisted(this))
{
this.cancelHighLightText();
if (!_oldVersion)
{
LayerManager.bin.addLayer(this);
this.moveObjectTo(LayerManager.bin.safeUpdatePos);
LayerManager.bin.updateObjectsOrder();
LayerManager.bin.updateIndex(LayerManager.bin.safeUpdatePos);
LayerManager.bin.updateObjectsPositions(LayerManager.bin.safeUpdatePos);
}
}
_isLocked = false;
return;
}
else if (graspedTime >= quickGraspTimeMin && handle != null)
{
Hand h = interactionBehaviour.closestHoveringHand;
handle.updateObjectsOrder();
if (_oldVersion)
{
handle.updateObjectsPositions(worldObject.transform.position);
}
else
{
handle.updateObjectsPositions(UnityVectorExtension.ToVector3(h.PalmPosition));
}
changeSelectState();
if (IsSelected)
{
selectActions();
disableGraspMovement();
//disableGraspActions();
if (LayerManager.goSelected == null)
{
LayerManager.goSelected = this;
}
else if (LayerManager.goSelected != this)
{
//LayerManager.goSelected.reColorWorldObj(Color.white);
LayerManager.goSelected.unselectActions();
LayerManager.goSelected = this;
}
}
else
{
unselectActions();
//enableGraspMovement();
//enableGraspActions();
LayerManager.goSelected = null;
}
}
/*
* if (handle != null && !IsSelected)
* {
* Hand h = interactionBehaviour.closestHoveringHand;
* handle.updateObjectsPositions(UnityVectorExtension.ToVector3(h.PalmPosition));
* }*/
_isLocked = false;
}
private void generateFeatureSet()
{
Controller controller = new Controller();
if (controller.IsConnected)
{
Frame frame = controller.Frame();
numberOfFingers = 0;
List <Hand> hands = frame.Hands;
for (int h = 0; h < frame.Hands.Count; h++)
{
Hand leapHand = frame.Hands[h];
Vector handXBasis = leapHand.PalmNormal.Cross(leapHand.Direction).Normalized;
Vector handYBasis = -leapHand.PalmNormal;
Vector handZBasis = -leapHand.Direction;
Vector handOrigin = leapHand.PalmPosition;
Matrix handTransform = new Matrix(handXBasis, handYBasis, handZBasis, handOrigin);
handTransform = handTransform.RigidInverse();
numberOfFingers += hands[h].Fingers.Count;
if (leapHand.IsLeft)
{
currentHand = "100";
}
if (leapHand.IsRight)
{
currentHand = "101";
}
fingerPositionString += currentHand + ",";
if (leapHand.Fingers.Count > 0)
{
writeToFile = true;
for (int f = 0; f < leapHand.Fingers.Count; f++)
{
Finger leapFinger = leapHand.Fingers[f];
Vector transformedPosition = handTransform.TransformPoint(leapFinger.TipPosition);
Vector transformedDirection = handTransform.TransformDirection(leapFinger.Direction);
float fingerLength = leapFinger.Length;
Finger nextLeapFinger = null;
Vector nextFingerPosition = new Vector();
if (f == 0)
{
nextLeapFinger = leapHand.Fingers[1];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 1)
{
nextLeapFinger = leapHand.Fingers[2];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 2)
{
nextLeapFinger = leapHand.Fingers[3];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 3)
{
nextLeapFinger = leapHand.Fingers[4];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
else if (f == 4)
{
nextLeapFinger = leapHand.Fingers[0];
nextFingerPosition = handTransform.TransformPoint(nextLeapFinger.TipPosition);
}
float Dist = DistanceBetweenTwoPoints(UnityVectorExtension.ToVector3(transformedPosition), UnityVectorExtension.ToVector3(nextFingerPosition));
fingerPositionString += transformedPosition.x + "," + transformedPosition.y + "," + transformedPosition.z + ",";
fingerDistanceString += (int)Dist + ",";
}
}
else
{
Debug.Log("No Fingers Found");
writeToFile = false;
}
}
if (frame.Hands.Count > 0)
{
fingerPositionString += fingerDistanceString;
fingerDistanceString = "";
featureText = numberOfFingers + "," + fingerPositionString;
Debug.Log(featureText + currentGestureName);
AddFestureSetToFile();
fingerPositionString = "";
}
}
if (!controller.IsConnected)
{
Debug.Log("Connect Leap Motion Controller");
}
}
protected float angle2LeapVectors(Leap.Vector a, Leap.Vector b)
{
return(Vector3.Angle(UnityVectorExtension.ToVector3(a), UnityVectorExtension.ToVector3(b)));
}
void PredictGesture(Controller c, int h)
{
if (DataUnderObservation == null && TwoDataUnderObservation == null)
{
return;
}
if (svm == null && twoSvm == null)
{
return;
}
//Try Catch Block For Prediction of gesture
try
{
for (int i = 0; i < DataUnderObservation.Rows; i++)
{
Matrix <float> row = DataUnderObservation.GetRow(i);
float predict = 0;
if (h == 1)
{
predict = svm.Predict(row);
}
else if (h == 2)
{
predict = twoSvm.Predict(row);
}
switch (predict)
{
case 0:
recognizedGesture = "LeftHandSlide";
break;
case 1:
recognizedGesture = "LeftHandClose";
break;
case 2:
recognizedGesture = "RightHandSlide";
break;
case 3:
recognizedGesture = "RightHandClose";
break;
case 4:
recognizedGesture = "RightHandGrip";
break;
case 5:
recognizedGesture = "LeftHandGrip";
break;
case 6:
recognizedGesture = "TwoHandsSlide";
break;
case 7:
recognizedGesture = "TwoHandsGrip";
break;
case 8:
recognizedGesture = "LeftCloseRightGrip";
break;
case 9:
recognizedGesture = "RightCloseLeftGrip";
break;
default:
recognizedGesture = "No Gesture Found";
break;
}
Debug.Log("Predicted Label: " + recognizedGesture);
}
}
catch (Exception ex)
{
Debug.Log(ex.Message);
recognizedGesture = "No Gesture Found";
}
if (recognizedGesture.Equals("LeftHandSlide") || recognizedGesture.Equals("RightHandSlide"))
{
Frame frame = c.Frame();
Hand hand = frame.Hands[0];
Vector palmPos = hand.PalmPosition;
if (previousFrameHandPosition.x == 0)
{
previousFrameHandPosition = palmPos;
return;
}
else
{
float diffX = (palmPos.x - previousFrameHandPosition.x) * speed;
float diffY = (palmPos.y - previousFrameHandPosition.y) * speed;
float diffZ = (palmPos.z - previousFrameHandPosition.z) * speed;
model3D.transform.position = new Vector3((xx + diffX), (yy + diffY), (zz + diffZ));
}
}
if (recognizedGesture.Equals("TwoHandsSlide"))
{
Frame frame = c.Frame();
for (int i = 0; i < frame.Hands.Count; i++)
{
Hand hand = frame.Hands[i];
float x = hand.PalmVelocity.x;
if (x < 0)
{
x = (-1) * x;
}
if (x > 40)
{
Vector palmPos = hand.PalmPosition;
if (previousFrameHandPosition == null)
{
previousFrameHandPosition = palmPos;
return;
}
else
{
float diffX = (palmPos.x - previousFrameHandPosition.x) * speed;
float diffY = (palmPos.y - previousFrameHandPosition.y) * speed;
float diffZ = (palmPos.z - previousFrameHandPosition.z) * speed;
model3D.transform.position = new Vector3((xx + diffX), (yy + diffY), (zz + diffZ));
}
}
}
}
if (recognizedGesture.Equals("LeftHandGrip") || recognizedGesture.Equals("RightHandGrip"))
{
previousFrameHandPosition = new Vector(0, 0, 0);
Frame frame = c.Frame();
Hand hand = frame.Hands[0];
float hPitch = hand.Direction.Pitch * 30;
float hYaw = hand.Direction.Yaw * 30;
float hRoll = hand.Direction.Roll * 30;
//model3D.transform.rotation = Quaternion.Euler((hPitch + x), (hYaw + y), (hRoll + z));
//x += model3D.transform.rotation.x;
//y += model3D.transform.rotation.y;
//z += model3D.transform.rotation.z;
}
if (recognizedGesture.Equals("TwoHandsGrip"))
{
previousFrameHandPosition = new Vector(0, 0, 0);
Frame frame = c.Frame();
Hand hand = frame.Hands[0];
Hand hand2 = frame.Hands[1];
Vector palm = hand.PalmPosition;
Vector palm2 = hand2.PalmPosition;
float dist = DistanceBetweenTwoPoints(UnityVectorExtension.ToVector3(palm), UnityVectorExtension.ToVector3(palm2));
currentDist = dist * 0.001f;
if (prevDist == 0.0f)
{
prevDist = currentDist;
}
else
{
float n = (currentDist - prevDist) * 3f;
Vector3 pos = model3D.transform.localScale;
pos.x += n;
pos.y += n;
pos.z += n;
model3D.transform.localScale = new Vector3(pos.x, pos.y, pos.z);
prevDist = currentDist;
}
}
if (recognizedGesture.Equals("LeftHandClose") || recognizedGesture.Equals("RightHandClose"))
{
previousFrameHandPosition = new Vector(0, 0, 0);
}
if (recognizedGesture.Equals("LeftCloseRightGrip "))
{
previousFrameHandPosition = new Vector(0, 0, 0);
}
if (recognizedGesture.Equals("RightCloseLeftGrip"))
{
previousFrameHandPosition = new Vector(0, 0, 0);
}
}
private void GestureListener()
{ //not editable when building
if (this.Lock)
{
return;
}
else
{
controller = new Controller();
Frame frame = _leapProvider.CurrentFrame;
List <Hand> hands = frame.Hands;
Hand leftHand = new Hand();
bool leftHandExist = false;
Hand rightHand = new Hand();
bool rightHandExist = false;
foreach (Hand hand in hands)
{
if (hand.IsLeft)
{
leftHand = hand;
leftHandExist = true;
}
else
{
rightHand = hand;
rightHandExist = true;
}
}
//One-Hand Gestures for Left Hand
if (leftHandExist)
{
//LeapGestureModel.State stateGrabLH = gestureGrabLH.Check(leftHand, null, Time.deltaTime);
//LeapGestureModel.State stateGrabbingLH = gestureGrabbingLH.Check(leftHand, null, Time.deltaTime);
LeapGestureModel.State statePinchLH = gesturePinchLH.Check(leftHand, null, Time.deltaTime);
//LeapGestureModel.State statePinchingLH = gesturePinchingLH.Check(leftHand, null, Time.deltaTime);
//LeapGestureModel.State stateTappingLH = gestureTappingLH.Check(leftHand, null, Time.deltaTime);
LeapGestureModel.State stateHoveringLH = gestureHoveringLH.Check(leftHand, null, Time.deltaTime);
/*
* if (stateGrabLH == LeapGestureModel.State.End)
* {
* OneHandGrab(0);
* }
* if (stateGrabbingLH == LeapGestureModel.State.InProcess)
* {
* OneHandGrabbing(0);
* }
* if (stateGrabbingLH == LeapGestureModel.State.End)
* {
* OneHandEndGrabbing(0);
* }*/
if (statePinchLH == LeapGestureModel.State.End)
{
OneHandPinch(0);
}/*
* if (statePinchingLH == LeapGestureModel.State.InProcess)
* {
* //OneHandPinching(0);
* }
* if (statePinchingLH== LeapGestureModel.State.End)
* {
* //OneHandEndPinching(0);
* }
* if (stateTappingLH == LeapGestureModel.State.InProcess)
* {
* //OneHandTapping(0);
* }
* if (stateTappingLH == LeapGestureModel.State.End)
* {
* // OneHandEndTapping(0);
* }*/
if (stateHoveringLH == LeapGestureModel.State.InProcess)
{
OneHandHovering(0);
}
}
//One-Hand Gestures for Right Hand
if (rightHandExist)
{
//LeapGestureModel.State stateGrabRH = gestureGrabRH.Check(null, rightHand, Time.deltaTime);
//LeapGestureModel.State stateGrabbingRH = gestureGrabbingRH.Check(null, rightHand, Time.deltaTime);
LeapGestureModel.State statePinchRH = gesturePinchRH.Check(null, rightHand, Time.deltaTime);
//LeapGestureModel.State statePinchingRH = gesturePinchingRH.Check(null, rightHand, Time.deltaTime);
//LeapGestureModel.State stateTappingRH = gestureTappingRH.Check(null, rightHand, Time.deltaTime);
LeapGestureModel.State stateHoveringRH = gestureHoveringRH.Check(null, rightHand, Time.deltaTime);
/*if (stateGrabRH == LeapGestureModel.State.End)
* {
* OneHandGrab(1);
* }
* if (stateGrabbingRH == LeapGestureModel.State.InProcess)
* {
* OneHandGrabbing(1);
* }
* if (stateGrabbingRH == LeapGestureModel.State.End)
* {
* OneHandEndGrabbing(1);
* }*/
if (statePinchRH == LeapGestureModel.State.End)
{
OneHandPinch(1);
}
/*if (statePinchingRH == LeapGestureModel.State.InProcess)
* {
* //OneHandPinching(1);
* }
* if (statePinchingRH == LeapGestureModel.State.End)
* {
* //OneHandEndPinching(1);
* }
* if (stateTappingRH == LeapGestureModel.State.InProcess)
* {
* //OneHandTapping(1);
* }
* if (stateTappingRH == LeapGestureModel.State.End)
* {
* //OneHandEndTapping(1);
* }*/
if (stateHoveringRH == LeapGestureModel.State.InProcess)
{
OneHandHovering(1);
}
}
if (leftHandExist && rightHandExist)
{
LeapGestureModel.State stateSplit = gestureSplit.Check(leftHand, rightHand, Time.deltaTime);
if (stateSplit == LeapGestureModel.State.End)
{
handle.moveObjectTo(UnityVectorExtension.ToVector3(leftHand.PalmPosition));
if (!oldVersion)
{
bin.moveObjectTo(UnityVectorExtension.ToVector3(rightHand.PalmPosition));
}
}
}
if (!leftHandExist && !rightHandExist)
{
OneHandHovering(2);
}
}
}
//to be override as detecting the start signal of a gusture
public override bool checkStart()
{
//1. Only one hand input
//2. The fingers have moved around hand
Hand hand = new Hand();
if ((leftHand == null) == (rightHand == null))
{
//Debug.Log("(Tapping)Start hand number problem ");
return(false);
}
else
{
if (leftHand != null)
{
hand = leftHand;
}
else
{
hand = rightHand;
}
}
try
{
Vector3 thumbTip = new Vector3();
Vector3 indexTip = new Vector3();
foreach (Finger finger in hand.Fingers)
{
if (finger.Type == Finger.FingerType.TYPE_THUMB)
{
thumbTip = finger.TipPosition.ToVector3();
}
else if (finger.Type == Finger.FingerType.TYPE_INDEX)
{
float angle = Vector3.Angle(hand.Direction.ToVector3(), finger.Direction.ToVector3());
if (angle > 60.0f)
{
Debug.Log("(Tapping)Start index finger direction problem ");
return(false);
}
indexTip = finger.TipPosition.ToVector3();
_lastTipPos = UnityVectorExtension.ToVector3(finger.TipPosition);
}
else
{
float angle = Vector3.Angle(hand.Direction.ToVector3(), finger.Direction.ToVector3());
if (angle < 120.0f)
{
Debug.Log("(Tapping)Start other fingers direction problem ");
return(false);
}
}
}
if (Vector3.Distance(thumbTip, indexTip) > hand.PalmWidth)
{
Debug.Log("(Tapping)Gesture start");
return(true);
}
else
{
Debug.Log("(Tapping)Gesture start problem");
return(false);
}
}
catch (Exception e)
{
Debug.Log(e.ToString());
return(false);
}
}
void Update()
{
Transform camera = Camera.main.transform;
Ray ray = new Ray(camera.position, Vector3.Normalize(camera.rotation * Vector3.forward));
RaycastHit hit;
mount_panel.text = setMountPanel();
if (cubeCanvas != null && geometryTargetObj != null)
{
cubeCanvas.transform.rotation = Quaternion.identity;
cubeCanvas.transform.position = geometryTargetObj.transform.position;
cubeCanvas.transform.Translate(0, 0, -0.05f);
}
if (left_hand.isActiveAndEnabled && right_hand.isActiveAndEnabled)
{
if (left_hand.GetLeapHand().GrabStrength > 0.99 && right_hand.GetLeapHand().GrabStrength > 0.99)
{
grabMode = GrabMode.Both;
}
else if (left_hand.GetLeapHand().GrabStrength > 0.99)
{
grabMode = GrabMode.LeftOnly;
}
else if (right_hand.GetLeapHand().GrabStrength > 0.99)
{
grabMode = GrabMode.RightOnly;
}
else
{
grabMode = GrabMode.Empty;
}
}
switch (grabMode)
{
case GrabMode.Empty:
if (Physics.Raycast(ray, out hit)) // Hit
{
if (hitObject == hit.collider.gameObject) // 같은 블럭
{
if (grabMode == GrabMode.Empty)
{
hitObject.transform.localScale = new Vector3(1f, 1f, 1f);
}
if (!cubeAlphaObj.activeSelf)
{
countdown = timeToSelect;
cubeAlphaObj.SetActive(true);
}
else
{
countdown -= Time.deltaTime;
if (countdown < 0.0f)
{
cubeCanvas.SetActive(true);
countdown = timeToSelect;
}
}
}
else if (hitObject != null) // 다른 블럭
{
if (blockToggle.isOn)
{
//nothing
}
else
{
cubeAlphaObj.SetActive(false);
cubeCanvas.SetActive(false);
cubeAlphaObj = null;
cubeCanvas = null;
blockToggle = null;
hitObject = hit.collider.gameObject;
cubeAlphaObj = hitObject.transform.GetChild(6).gameObject;
cubeCanvas = hitObject.transform.GetChild(7).gameObject;
blockToggle = cubeCanvas.transform.GetChild(0).GetChild(0).GetComponent <Toggle>();
}
}
else // null 블럭
{
hitObject = hit.collider.gameObject;
cubeAlphaObj = hitObject.transform.GetChild(6).gameObject;
cubeCanvas = hitObject.transform.GetChild(7).gameObject;
blockToggle = cubeCanvas.transform.GetChild(0).GetChild(0).GetComponent <Toggle>();
}
}
else // no Hit
{
hitObject.transform.localScale = new Vector3(1f, 1f, 1f);
if (hitObject != null) // previous hit block exist
{
if (blockToggle.isOn)
{
//nothing
}
else
{
//hitObject = null, effect = false&null
hitObject = null;
cubeAlphaObj.SetActive(false);
cubeCanvas.SetActive(false);
cubeAlphaObj = null;
cubeCanvas = null;
}
}
else
{
}
}
break;
case GrabMode.LeftOnly:
Debug.Log("left");
if (Vector3.Distance(hitObject.gameObject.transform.position, UnityVectorExtension.ToVector3(left_hand.GetLeapHand().Fingers[0].TipPosition)) < 0.15f && !blockToggle.isOn)
{
hitObject.transform.localScale = new Vector3(0.8f, 0.8f, 0.8f);
Vector3 newPosition = UnityVectorExtension.ToVector3(left_hand.GetLeapHand().Fingers[0].TipPosition);
hitObject.transform.position = new Vector3(truncate(newPosition.x), truncate(newPosition.y), truncate(newPosition.z));
}
break;
case GrabMode.RightOnly:
Debug.Log("right");
if (Vector3.Distance(hitObject.gameObject.transform.position, UnityVectorExtension.ToVector3(right_hand.GetLeapHand().Fingers[0].TipPosition)) < 0.15f && !blockToggle.isOn)
{
hitObject.transform.localScale = new Vector3(0.8f, 0.8f, 0.8f);
Vector3 newPosition = UnityVectorExtension.ToVector3(right_hand.GetLeapHand().Fingers[0].TipPosition);
hitObject.transform.position = new Vector3(truncate(newPosition.x), truncate(newPosition.y), truncate(newPosition.z));
}
break;
case GrabMode.Both:
if (Vector3.Dot(UnityVectorExtension.ToVector3(left_hand.GetLeapHand().PalmNormal), Vector3.up) > 0 && Vector3.Dot(UnityVectorExtension.ToVector3(right_hand.GetLeapHand().PalmNormal), Vector3.up) > 0)
{
block_panel.SetActive(true);
}
//float palm_distance = Vector3.Distance(UnityVectorExtension.ToVector3(left_hand.GetLeapHand().PalmPosition), UnityVectorExtension.ToVector3(right_hand.GetLeapHand().PalmPosition));
//hitObject.transform.localScale = new Vector3(truncate(palm_distance), truncate(palm_distance), truncate(palm_distance));
break;
}
}
protected void onGraspStayRemoved()
{
_isLocked = true;
graspedTime += Time.deltaTime;
if (interactionBehaviour.closestHoveringHand == null)
{
return;
}
if (graspedTime < quickGraspTimeMax)
{
_isLocked = false;
return;
}
else
{
if (!_isLongGrasped)
{
_isLongGrasped = true;
unselectActions();
enableGraspMovement();
//enableGraspActions();
LayerManager.goSelected = null;
}
}
bool inRange = LayerManager.handle.updateLayerExisted(this);
if (LayerManager.layerMetaphorsDisActived.Contains(this))
{
if (inRange)
{
LayerManager.layerMetaphorsDisActived.Remove(this);
LayerManager.layerMetaphorsActived.Add(this);
if (_oldVersion)
{
LayerManager.handle.updateIndex(worldObject.transform.position);
}
else
{
LayerManager.handle.updateIndex(LayerManager.handle.safeUpdatePos);
}
this.getLayer().show();
if (!_oldVersion)
{
LayerManager.bin.removeLayer(this);
this.getLayer().getDuplication().showSelf();
this.getLayer().getDuplication().hideText();
this.worldObject.GetComponent <MeshRenderer>().enabled = false;
}
}
else
{
if (handle != null && !IsSelected)
{
Hand h = interactionBehaviour.closestHoveringHand;
if (_oldVersion)
{
handle.updateIndex(worldObject.transform.position);
}
else
{
handle.updateIndex(UnityVectorExtension.ToVector3(h.PalmPosition));
}
}
}
}
else
{
if (inRange)
{
if (handle != null && !IsSelected)
{
Hand h = interactionBehaviour.closestHoveringHand;
if (_oldVersion)
{
handle.updateIndex(worldObject.transform.position);
}
else
{
handle.updateIndex(UnityVectorExtension.ToVector3(h.PalmPosition));
}
}
}
else
{
LayerManager.layerMetaphorsActived.Remove(this);
LayerManager.layerMetaphorsDisActived.Add(this);
this.getLayer().hide();
if (!_oldVersion)
{
this.getLayer().getDuplication().hideSelf();
this.worldObject.GetComponent <MeshRenderer>().enabled = true;
}
}
}
if (handle != null && !IsSelected)
{
Hand h = interactionBehaviour.closestHoveringHand;
handle.updateObjectsOrder();
if (_oldVersion)
{
handle.updateObjectsPositions(worldObject.transform.position);
}
else
{
handle.updateObjectsPositions(UnityVectorExtension.ToVector3(h.PalmPosition));
}
}
_isLocked = false;
}