/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
using (UpdateHandDataPerfMarker.Auto())
{
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
SpatialInteractionSourceState sourceState = interactionSourceState.source.GetSpatialInteractionSourceState();
if (sourceState == null)
{
return;
}
#if WINDOWS_UWP
handMeshProvider?.UpdateHandMesh(sourceState);
#endif // WINDOWS_UWP
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
Vector3 jointPosition = jointPoses[i].Position.ToUnityVector3();
Quaternion jointOrientation = jointPoses[i].Orientation.ToUnityQuaternion();
// We want the joints to follow the playspace, so fold in the playspace transform here to
// put the joint pose into world space.
jointPosition = MixedRealityPlayspace.TransformPoint(jointPosition);
jointOrientation = MixedRealityPlayspace.Rotation * jointOrientation;
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (handJoint == TrackedHandJoint.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
unityJointPoses[handJoint] = new MixedRealityPose(jointPosition, jointOrientation);
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
#if WINDOWS_UWP
handDefinition?.UpdateHandMesh(sourceState);
#endif // WINDOWS_UWP
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
Vector3 jointPosition = jointPoses[i].Position.ToUnityVector3();
Quaternion jointOrientation = jointPoses[i].Orientation.ToUnityQuaternion();
// We want the joints to follow the playspace, so fold in the playspace transform here to
// put the joint pose into world space.
jointPosition = MixedRealityPlayspace.TransformPoint(jointPosition);
jointOrientation = MixedRealityPlayspace.Rotation * jointOrientation;
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (handJoint == TrackedHandJoint.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
unityJointPoses[handJoint] = new MixedRealityPose(jointPosition, jointOrientation);
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
break;
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
public void ChangeSaveAnimationState(
string animName = "",
HandPose.HandPoseType selectedHandAnim = HandPose.HandPoseType.NoSelection,
AnimationClip handPoseAnim = null)
{
this.animName = animName;
this.selectedHandAnim = selectedHandAnim;
if (handPoseAnim is null)
{
handPoseAnim = HandPose.GetHandAnimationClip(this.selectedHandAnim);
}
this.handPoseAnim = handPoseAnim;
}
private HandPose GenerateOpenFingersPose(string name, Finger[] fingers)
{
var nonThumbOtherFingers = (new[] { Finger.Index, Finger.Middle, Finger.Ring, Finger.Pinky }).Except(fingers);
var fingersOpenPose = new HandPose(name, new PalmPose(new AnyHandContext(), PoseDirection.Forward, PoseDirection.Up),
new FingerPose(fingers, FingerFlexion.Open, PoseDirection.Up),
new FingertipDistanceRelation(fingers, RelativeDistance.NotTouching, nonThumbOtherFingers.Union(new[] { Finger.Thumb })));
if (nonThumbOtherFingers.Any())
{
fingersOpenPose.PoseConstraints.Add(new FingerPose(nonThumbOtherFingers, PoseDirection.Down | PoseDirection.Backward | PoseDirection.Forward | PoseDirection.Left | PoseDirection.Right));
}
return(fingersOpenPose);
}
/// <summary>
/// Calculate the best pre-recorded snap-point to grab an object.
/// </summary>
/// <param name="grabbable">The snappable object.</param>
/// <returns>
/// If the snappable is object is valid, the best SnapPoint to grab it alongside the
/// Hand-Pose to use when grabbing at that position.
/// </returns>
public ObjectSnappingAddress SnapForGrabbable(GameObject grabbable)
{
if (grabbable != null &&
grabbable.TryGetComponent <Snappable>(out Snappable snappable))
{
HandPose userPose = this.puppet.TrackedPose(snappable.transform);
BaseSnapPoint snapPose = snappable.FindBestSnapPose(userPose, out ScoredHandPose bestPose);
if (snapPose != null)
{
return(new ObjectSnappingAddress(snappable, snapPose, bestPose));
}
}
return(null);
}
public void ChangeSaveAnimationState(
string animName = "",
int selectStateIndex = 0,
AnimationClip handPoseAnim = null)
{
this.animName = animName;
// TODO: 一時対処
this.selectedHandAnim = HandPose.HandPoseType.NoSelection;
if (handPoseAnim is null)
{
handPoseAnim = HandPose.GetHandAnimationClip(this.selectedHandAnim);
}
this.handPoseAnim = handPoseAnim;
}
/// <summary>
/// Relay to the Puppet to set the ghost hand to the desired static pose
/// </summary>
/// <param name="handGrabPoint">The point to read the HandPose from</param>
public void SetPose(HandGrabPoint handGrabPoint)
{
HandPose userPose = handGrabPoint.HandPose;
if (userPose == null)
{
return;
}
Transform relativeTo = handGrabPoint.RelativeTo;
_puppet.SetJointRotations(userPose.JointRotations);
SetGripPose(handGrabPoint.RelativeGrip, relativeTo);
}
public async Task Init()
{
// Step1: Define the Rock-Paper-Scissors gestures
// Create a pose for 'Rock'...
var rockPose = new HandPose("RockPose", new FingerPose(new AllFingersContext(), FingerFlexion.Folded));
rockPose.Triggered += (s, arg) => UserStrategyChanged?.Invoke(GameStrategy.Rock);
// ...another for 'Paper'...
var paperPose = new HandPose("PaperPose", new PalmPose(new AnyHandContext(), PoseDirection.Left | PoseDirection.Right, PoseDirection.Forward),
new FingerPose(new AllFingersContext(), FingerFlexion.Open));
paperPose.Triggered += (s, arg) => UserStrategyChanged?.Invoke(GameStrategy.Paper);
// ...and last one for 'Scissors'...
var scissorsPose = new HandPose("ScissorsPose", new FingerPose(new[] { Finger.Index, Finger.Middle }, FingerFlexion.Open),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Middle),
new FingerPose(new[] { Finger.Ring, Finger.Pinky }, FingerFlexion.Folded));
scissorsPose.Triggered += (s, arg) => UserStrategyChanged?.Invoke(GameStrategy.Scissors);
// ...a PassThroughtGestureSegment is a structural gesture segment that provides a way to simplify a gesture state machine construction by 'short-circuiting'
// between gesture segments connectd to it and gesture segements it connects to. It helps reduce the number of SubPaths that needs to be defined.
// Very handy when you need to define a Clique (see https://en.wikipedia.org/wiki/Clique_(graph_theory)#1)
// as in this case where Rock, Paper and Scissors are all connected to each other...
var epsilonState = new PassThroughGestureSegment("Epsilon");
// ...this pose is an artificial stop pose. Namely, we want to keep the gesture detector in one of the pose states without ending the gesture so we add this
// pose as a pose that completes the gesture assuming the user will not perform it frequently.
// As long as the user continues to perform the 'Rock', 'Paper' or 'Scissors' poses we will remain within the gesture's state machine.
var giveUpPose = new HandPose("GiveUpPose", new PalmPose(new AnyHandContext(), PoseDirection.Forward, PoseDirection.Up),
new FingerPose(new AllFingersContext(), FingerFlexion.Open));
_gameGesture = new Gesture("RockPaperScissorGesture", epsilonState, giveUpPose);
// ...add a sub path back and forth from the PassthroughGestureSegment to the various poses
_gameGesture.AddSubPath(epsilonState, rockPose, epsilonState);
_gameGesture.AddSubPath(epsilonState, paperPose, epsilonState);
_gameGesture.AddSubPath(epsilonState, scissorsPose, epsilonState);
// In case the user performs a pose that is not one of the game poses the gesture resets and this event will trigger
_gameGesture.IdleTriggered += (s, arg) => UserStrategyChanged?.Invoke(GameStrategy.None);
// Step2: Connect to Gesture Detection Service, route StatusChanged event to the UI and register the gesture
_gesturesService = GesturesServiceEndpointFactory.Create();
_gesturesService.StatusChanged += (oldStatus, newStatus) => GesturesDetectionStatusChanged?.Invoke(oldStatus, newStatus);
await _gesturesService.ConnectAsync();
await _gesturesService.RegisterGesture(_gameGesture);
}
// It defines and registers the Rotate Right Gesture.
private static async Task RegisterRotateLeftGesture()
{
var hold = new HandPose("Hold", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Right, Finger.Thumb));
var rotate = new HandPose("Rotate", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Above, Finger.Thumb));
_rotateLeftGesture = new Gesture("RotateLeft", hold, rotate);
_rotateLeftGesture.Triggered += (s, e) => OnGestureDetected(s, e, ConsoleColor.Yellow);
await _gesturesService.RegisterGesture(_rotateLeftGesture, isGlobal : true);
}
private Gesture GenerateSwipeGesure(string name, PoseDirection direction)
{
var fingers = new[] { Finger.Index, Finger.Middle, Finger.Ring };
var fingerSet = new HandPose("FingersSet", new PalmPose(new AnyHandContext(), direction, orientation: PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Middle, RelativeDistance.Touching, new[] { Finger.Index, Finger.Ring }),
new FingerPose(fingers, PoseDirection.Forward));
var fingersBent = new HandPose("FingersBent", new PalmPose(new AnyHandContext(), direction, orientation: PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Middle, RelativeDistance.Touching, new[] { Finger.Index, Finger.Ring }),
new FingerPose(fingers, direction | PoseDirection.Backward));
var swipeGesture = new Gesture(name, fingerSet, fingersBent);
return(swipeGesture);
}
private async Task RegisterThankYouGesture()
{
//var forw = new PalmMotion("forward", MotionPlane.Depth);
var thankyou1 = new HandPose(
"thankyou1",
new PalmPose(Hand.RightHand, PoseDirection.Up | PoseDirection.Backward),
new FingerPose(new[] { Finger.Ring, Finger.Pinky, Finger.Middle, Finger.Index, Finger.Thumb }, FingerFlexion.Open),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.Touching, Finger.Middle),
new FingertipDistanceRelation(Finger.Thumb, RelativeDistance.Touching, Finger.Index));
var thxGesture = new Gesture("thxgesture", thankyou1);
thxGesture.Triggered += (s, arg) => GestureChanged?.Invoke(arg.GestureSegment.Name);
await _gesturesService.RegisterGesture(thxGesture, isGlobal : true);
}
private HandPose TrackedPose()
{
if (!_handGrabInteractor.Hand.GetJointPosesLocal(out ReadOnlyHandJointPoses localJoints))
{
return(null);
}
HandPose result = new HandPose(_handGrabInteractor.Hand.Handedness);
for (int i = 0; i < FingersMetadata.HAND_JOINT_IDS.Length; ++i)
{
HandJointId jointID = FingersMetadata.HAND_JOINT_IDS[i];
result.JointRotations[i] = localJoints[jointID].rotation;
}
return(result);
}
/// <summary>
/// Find the Snap Point on this Snappabe with the best score to the given hand pose.
/// </summary>
/// <param name="userPose">The hand pose to be compared with this snappable.</param>
/// <param name="bestHandPose">The most similar Hand Pose that snaps to the object, with its score.</param>
/// <returns>The SnapPoint that is closer to the hand</returns>
public BaseSnapPoint FindBestSnapPose(HandPose userPose, out ScoredHandPose bestHandPose)
{
BaseSnapPoint bestSnap = null;
bestHandPose = ScoredHandPose.Null();
foreach (var snapPose in this.snapPoints)
{
ScoredHandPose pose = snapPose.CalculateBestPose(userPose);
if (pose.Score > bestHandPose.Score)
{
bestSnap = snapPose;
bestHandPose = pose;
}
}
return(bestSnap);
}
public static Pose Interpolate(Pose from, Pose to, float time)
{
return(new Pose {
headPosition = Vector3.Lerp(from.headPosition, to.headPosition, time),
headRotation = Quaternion.Slerp(from.headRotation, to.headRotation, time),
handLeftPosition = Vector3.Lerp(from.handLeftPosition, to.handLeftPosition, time),
handLeftRotation = Quaternion.Slerp(from.handLeftRotation, to.handLeftRotation, time),
handRightPosition = Vector3.Lerp(from.handRightPosition, to.handRightPosition, time),
handRightRotation = Quaternion.Slerp(from.handRightRotation, to.handRightRotation, time),
voiceAmplitude = Mathf.Lerp(from.voiceAmplitude, to.voiceAmplitude, time),
controllerLeftPose = ControllerPose.Interpolate(from.controllerLeftPose, to.controllerLeftPose, time),
controllerRightPose = ControllerPose.Interpolate(from.controllerRightPose, to.controllerRightPose, time),
handLeftPose = HandPose.Interpolate(from.handLeftPose, to.handLeftPose, time),
handRightPose = HandPose.Interpolate(from.handRightPose, to.handRightPose, time),
});
}
void CollectInput(ref HandPose hand, ref ControllerPose controller, ref HandInput i)
{
i.handTrigger = hand.gripFlex;
i.previousIndexTrigger = i.indexTrigger;
i.indexTrigger = hand.indexFlex;
if (i.indexTrigger >= IndexThreshold && i.previousIndexTrigger < IndexThreshold)
{
i.indexPressFrame = context.renderFrame;
}
i.isPointing = true;
i.isPressingX = controller.button1IsDown;
i.isPressingY = controller.button2IsDown;
i.stick = controller.joystickPosition;
}
private static async Task RegisterFourFingerGesture()
{
// Start with defining the poses:
var closedFist = new HandPose("ClosedFist", new FingerPose(new AllFingersContext(), FingerFlexion.Folded));
var fourFingers = new HandPose("FourFingers", new FingerPose(new[] { Finger.Index, Finger.Middle, Finger.Ring, Finger.Pinky }, FingerFlexion.Open, PoseDirection.Forward),
new FingerPose(new[] { Finger.Thumb }, FingerFlexion.Folded));
// Then define the gesture using the hand pose objects defined above forming a simple state machine: closedFist -> fourFingers
fourFingerGesture = new Gesture("FourFingers", closedFist, fourFingers);
fourFingerGesture.Triggered += (s, e) => OnGestureDetected(s, e, ConsoleColor.Green);
// Step 3: Register the gesture
// Registering the like gesture _globally_ (i.e. isGlobal:true), by global registration we mean this gesture will be
// detected even it was initiated not by this application or if the this application isn't in focus
await _gesturesService.RegisterGesture(fourFingerGesture, isGlobal : true);
}
private static async Task RegisterDropTheMicGesture()
{
// Our starting pose is a full fist pointing down and/or forward
var fist = new HandPose("Fist", new PalmPose(new AnyHandContext(), PoseDirection.Down | PoseDirection.Forward),
new FingerPose(new[] { Finger.Index, Finger.Middle, Finger.Ring }, FingerFlexion.Folded));
// Final pose is when the hand is "open" i.e. spread
var spread = new HandPose("Spread", new PalmPose(new AnyHandContext(), PoseDirection.Down | PoseDirection.Forward),
new FingerPose(new[] { Finger.Thumb, Finger.Index, Finger.Middle, Finger.Ring }, FingerFlexion.Open));
// ... finally define the gesture using the hand pose objects defined above forming a simple state machine: fist -> spread
_dropTheMicGesture = new Gesture("DropTheMic", fist, spread);
_dropTheMicGesture.Triggered += (s, e) => OnGestureDetected(s, e, ConsoleColor.Blue);
// Registering the like gesture _globally_ (i.e. isGlobal:true), by global registration we mean this gesture will be
// detected even it was initiated not by this application or if the this application isn't in focus
await _gesturesService.RegisterGesture(_dropTheMicGesture, isGlobal : true);
}
public async Task Init()
{
// Set the gesture service
_gesturesService = GesturesServiceEndpointFactory.Create("localhost");
_gesturesService.StatusChanged += (oldVal, newVal) => GesturesDetectionStatusChanged?.Invoke(oldVal, newVal);
await _gesturesService.ConnectAsync();
var hurufI = new HandPose(
"Huruf_I",
new PalmPose(Hand.RightHand, PoseDirection.Up | PoseDirection.Forward),
new FingerPose(new[] { Finger.Pinky }, FingerFlexion.Open),
new FingerPose(new[] { Finger.Index, Finger.Ring, Finger.Middle }, FingerFlexion.Folded));
hurufI.Triggered += (s, arg) => GestureChanged?.Invoke(arg.GestureSegment.Name);
var piece = new HandPose(
"Piece",
new PalmPose(Hand.RightHand, PoseDirection.Up | PoseDirection.Forward),
new FingerPose(new[] { Finger.Index, Finger.Middle }, FingerFlexion.Open),
new FingerPose(new[] { Finger.Pinky, Finger.Ring, }, FingerFlexion.Folded));
piece.Triggered += (s, arg) => GestureChanged?.Invoke(arg.GestureSegment.Name);
var baik = new HandPose(
"baik",
new PalmPose(Hand.RightHand, PoseDirection.Left | PoseDirection.Forward),
new FingerPose(new[] { Finger.Ring, Finger.Pinky, Finger.Middle }, FingerFlexion.OpenStretched),
new FingerPose(new[] { Finger.Index, Finger.Thumb, }, FingerFlexion.Folded));
baik.Triggered += (s, arg) => GestureChanged?.Invoke(arg.GestureSegment.Name);
var alphabetGesture = new PassThroughGestureSegment("performing_state");
_detectedGesture = new Gesture("Sample_Gesture", alphabetGesture);
_detectedGesture.AddSubPath(alphabetGesture, hurufI, alphabetGesture);
_detectedGesture.AddSubPath(alphabetGesture, piece, alphabetGesture);
_detectedGesture.AddSubPath(alphabetGesture, baik, alphabetGesture);
await _gesturesService.RegisterGesture(_detectedGesture, isGlobal : true);
await RegisterLikeGesture();
await RegisterThankYouGesture();
}
// It defines and registers the Rotate Right Gesture.
private static async Task RegisterRotateRightGesture()
{
// Start with defining the first pose
var hold = new HandPose("Hold", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Right, Finger.Thumb));
// define the second pose
var rotate = new HandPose("Rotate", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Below, Finger.Thumb));
// define the gesture using the hand pose objects defined above forming a simple state machine
_rotateRightGesture = new Gesture("RotateRight", hold, rotate);
_rotateRightGesture.Triggered += (s, e) => OnGestureDetected(s, e, ConsoleColor.Blue);
await _gesturesService.RegisterGesture(_rotateRightGesture, isGlobal : true);
}
/// <summary>
/// Writes the desired rotation values for each joint based on the provided SnapAddress.
/// Apart from the rotations it also writes in the syntheticHand if it should allow rotations
/// past that.
/// When no snap is provided, it frees all fingers allowing unconstrained tracked motion.
/// </summary>
private void UpdateFingers(HandPose handPose, HandFingerFlags grabbingFingers, float strength)
{
Quaternion[] desiredRotations = handPose.JointRotations;
_syntheticHand.OverrideAllJoints(desiredRotations, strength);
for (int fingerIndex = 0; fingerIndex < Constants.NUM_FINGERS; fingerIndex++)
{
int fingerFlag = 1 << fingerIndex;
JointFreedom fingerFreedom = handPose.FingersFreedom[fingerIndex];
if (fingerFreedom == JointFreedom.Constrained &&
((int)grabbingFingers & fingerFlag) != 0)
{
fingerFreedom = JointFreedom.Locked;
}
_syntheticHand.SetFingerFreedom((HandFinger)fingerIndex, fingerFreedom);
}
}
public static PoseFrame Interpolate(PoseFrame a, PoseFrame b, float t)
{
return(new PoseFrame
{
headPosition = Vector3.Lerp(a.headPosition, b.headPosition, t),
headRotation = Quaternion.Slerp(a.headRotation, b.headRotation, t),
handLeftPosition = Vector3.Lerp(a.handLeftPosition, b.handLeftPosition, t),
handLeftRotation = Quaternion.Slerp(a.handLeftRotation, b.handLeftRotation, t),
handRightPosition = Vector3.Lerp(a.handRightPosition, b.handRightPosition, t),
handRightRotation = Quaternion.Slerp(a.handRightRotation, b.handRightRotation, t),
voiceAmplitude = Mathf.Lerp(a.voiceAmplitude, b.voiceAmplitude, t),
controllerLeftPose = ControllerPose.Interpolate(a.controllerLeftPose, b.controllerLeftPose, t),
controllerRightPose = ControllerPose.Interpolate(a.controllerRightPose, b.controllerRightPose, t),
handLeftPose = HandPose.Interpolate(a.handLeftPose, b.handLeftPose, t),
handRightPose = HandPose.Interpolate(a.handRightPose, b.handRightPose, t),
});
}
private async Task RegisterLikeGesture()
{
// Our starting pose is a fist
var fist = new HandPose("Fist", new PalmPose(new AnyHandContext(), PoseDirection.Left | PoseDirection.Right),
new FingerPose(new AllFingersContext(), FingerFlexion.Folded));
// In the final pose the thumb flexion will open in the "up" direction
var like = new HandPose("Like", new PalmPose(new AnyHandContext(), PoseDirection.Left | PoseDirection.Right),
new FingerPose(new[] { Finger.Index, Finger.Middle, Finger.Ring, Finger.Pinky }, FingerFlexion.Folded),
new FingerPose(Finger.Thumb, FingerFlexion.Open, PoseDirection.Up));
// ... finally define the gesture using the hand pose objects defined above forming a simple state machine: fist -> Like
_likeGesture = new Gesture("LikeGesture", fist, like);
_likeGesture.Triggered += (s, arg) => GestureChanged?.Invoke(arg.GestureSegment.Name);
// Registering the like gesture _globally_ (i.e. isGlobal:true), by global registration we mean this gesture will be
// detected even it was initiated not by this application or if the this application isn't in focus
await _gesturesService.RegisterGesture(_likeGesture, isGlobal : true);
}
/// <summary>
/// Calculate the best pre-recorded snap-point to grab an object.
/// </summary>
/// <param name="grabbable">The snappable object.</param>
/// <returns>
/// If the snappable is object is valid, the best SnapPoint to grab it alongside the
/// Hand-Pose to use when grabbing at that position.
/// </returns>
private (BaseSnapPoint, ScoredHandPose)? SnapForGrabbable(GameObject grabbable)
{
if (grabbable == null)
{
return(null);
}
if (grabbable.TryGetComponent <Snappable>(out Snappable snappable))
{
HandPose userPose = this.puppet.TrackedPose(snappable.transform);
BaseSnapPoint snapPose = snappable.FindBestSnapPose(userPose, out ScoredHandPose bestPose);
if (snapPose != null)
{
return(snapPose, bestPose);
}
}
return(null);
}
/// <summary>
/// Interpolate between two ScoredHandPose. Both ScoredHandPoses must have the same direction.
/// This method does not only moves the hands, but also adjusts the score linearly.
/// </summary>
/// <param name="from">The base ScoredHandPose to interpolate from.</param>
/// <param name="to">The target ScoredHandPose to interpolate to.</param>
/// <param name="t">The interpolation factor, 0 for the base, 1 for the target value.</param>
/// <returns>A ScoredHandPose between base and target, null if they are not interpolable.</returns>
public static ScoredHandPose?Lerp(ScoredHandPose from, ScoredHandPose to, float t)
{
if (from.Direction != to.Direction)
{
UnityEngine.Debug.LogError("ScoredHandPose must have same direction for interpolation");
return(null);
}
float score = UnityEngine.Mathf.Lerp(from.Score, to.Score, t);
HandPose?pose = HandPose.Lerp(from.Pose, to.Pose, t);
if (!pose.HasValue)
{
UnityEngine.Debug.LogError("ScoredHandPose interpolation error");
return(null);
}
return(new ScoredHandPose(pose.Value, score, from.Direction));
}
private string GetHandPoseTriggerName(HandPose pose)
{
string poseName;
if (pose == HandPose.hold)
{
poseName = "Hold";
}
else if (pose == HandPose.pinch)
{
poseName = "Pinch";
}
else
{
poseName = "Relax";
}
return(poseName);
}
private static async Task RegisterRotateRightGesture()
{
// Start with defining the first pose, ...
var hold = new HandPose("Hold", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Above, Finger.Thumb));
// ... define the second pose, ...
var rotate = new HandPose("Rotate", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Right, Finger.Thumb));
// ... finally define the gesture using the hand pose objects defined above forming a simple state machine: hold -> rotate
_rotateGesture = new Gesture("RotateRight", hold, rotate);
_rotateGesture.Triggered += (s, e) => OnGestureDetected(s, e, ConsoleColor.Yellow);
// Step 3: Register the gesture
// Registering the like gesture _globally_ (i.e. isGlobal:true), by global registration we mean this gesture will be
// detected even it was initiated not by this application or if the this application isn't in focus
await _gesturesService.RegisterGesture(_rotateGesture, isGlobal : true);
}
private void Update()
{
if (Input.GetKeyDown(KeyCode.Space))
{
SavePose();
}
foreach (var pose in Poses)
{
Handedness handedness = ComparePose(pose);
if (handedness != Handedness.None)
{
currentPose = pose;
currentPose.handedness = handedness;
return;
}
}
currentPose = None;
}
private async void PageLoaded(object sender, RoutedEventArgs e)
{
// Step 1: Connect to Microsoft Gestures service
_gesturesService = GesturesServiceEndpointFactory.Create();
var dispatcher = CoreWindow.GetForCurrentThread().Dispatcher;
_gesturesService.StatusChanged += async(s, arg) => await dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => GesturesServiceStatus.Text = $"[{arg.Status}]");
Unloaded += async(s, arg) =>
{
await _gesturesService?.Disconnect();
_gesturesService?.Dispose();
};
await _gesturesService.ConnectAsync();
// Step 2: Define your custom gesture
// Start with defining the first pose, ...
var hold = new HandPose("Hold", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Above, Finger.Thumb));
// ... define the second pose, ...
var rotate = new HandPose("Rotate", new FingerPose(new[] { Finger.Thumb, Finger.Index }, FingerFlexion.Open, PoseDirection.Forward),
new FingertipDistanceRelation(Finger.Index, RelativeDistance.NotTouching, Finger.Thumb),
new FingertipPlacementRelation(Finger.Index, RelativePlacement.Right, Finger.Thumb));
// ... finally define the gesture using the hand pose objects defined above forming a simple state machine: hold -> rotate
_rotateGesture = new Gesture("RotateRight", hold, rotate);
_rotateGesture.Triggered += async(s, args) => await dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
var rotateTransform = new RotateTransform {
CenterX = Arrow.ActualWidth / 2, CenterY = Arrow.ActualHeight / 2
};
rotateTransform.Angle = ++_rotateTimes * 90;
Arrow.RenderTransform = rotateTransform;
});
// Step 3: Register the gesture (When window focus is lost (gained) the service will automatically unregister (register) the gesture)
// To manually control the gesture registration, pass 'isGlobal: true' parameter in the function call below
await _gesturesService.RegisterGesture(_rotateGesture);
}
private void UpdateAnimStates()
{
bool grabbing = _grabber.GrabbedObject != null;
HandPose grabPose = m_defaultGrabPose;
if (grabbing)
{
HandPose customPose = _grabber.GrabbedObject.GetComponent <HandPose>();
if (customPose != null)
{
grabPose = customPose;
}
}
// Pose
HandPoseId handPoseId = grabPose.PoseId;
m_animator.SetInteger(_animParamIndexPose, (int)handPoseId);
// Flex
// blend between open hand and fully closed fist
float flex = OVRInput.Get(OVRInput.Axis1D.PrimaryHandTrigger, m_controller);
m_animator.SetFloat(_animParamIndexFlex, flex);
// Point
bool canPoint = !grabbing || grabPose.AllowPointing;
float point = canPoint ? _pointBlend : 0.0f;
m_animator.SetLayerWeight(_animLayerIndexPoint, point);
// Thumbs up
bool canThumbsUp = !grabbing || grabPose.AllowThumbsUp;
float thumbsUp = canThumbsUp ? _thumbsUpBlend : 0.0f;
m_animator.SetLayerWeight(_animLayerIndexThumb, thumbsUp);
float pinch = OVRInput.Get(OVRInput.Axis1D.PrimaryIndexTrigger, m_controller);
m_animator.SetFloat("Pinch", pinch);
}
private void ChangeSelectionHandAnimation()
{
if (usePreviousAnimationOnHandAnimation)
{
var animController = originalAvatar.StandingAnimController;
var previousAnimation = animController[AnimationsGUI.HANDANIMS[(int)selectedHandAnim - 1]];
// 未設定でなければ以前設定されていたものをHandPoseAnimationとして使う
if (previousAnimation != null && previousAnimation.name != AnimationsGUI.HANDANIMS[(int)selectedHandAnim - 1])
{
handPoseAnim = previousAnimation;
}
else
{
handPoseAnim = HandPose.GetHandAnimationClip(selectedHandAnim);
}
}
else
{
handPoseAnim = HandPose.GetHandAnimationClip(selectedHandAnim);
}
}
public void OnRightHand(HandPose transition)
{
Moment now = HolofunkModel.Clock.Now;
lock (m_rightHandModel.StateMachine) {
Spam.Model.WriteLine("PlayerModel.OnRightHand: received transition " + transition);
m_rightHandModel.StateMachine.OnNext(LoopieEvent.FromHandPose(transition), now);
}
}