/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
Profiler.BeginSample("[MRTK] WindowsMixedRealityArticulatedHand.UpdateHandData");
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
#if WINDOWS_UWP
handDefinition?.UpdateHandMesh(sourceState);
#endif // WINDOWS_UWP
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
Vector3 jointPosition = jointPoses[i].Position.ToUnityVector3();
Quaternion jointOrientation = jointPoses[i].Orientation.ToUnityQuaternion();
// We want the joints to follow the playspace, so fold in the playspace transform here to
// put the joint pose into world space.
jointPosition = MixedRealityPlayspace.TransformPoint(jointPosition);
jointOrientation = MixedRealityPlayspace.Rotation * jointOrientation;
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (handJoint == TrackedHandJoint.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
unityJointPoses[handJoint] = new MixedRealityPose(jointPosition, jointOrientation);
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
break;
}
}
Profiler.EndSample(); // UpdateHandData
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
private bool GetHistoricalPose(out Vector3 cameraPosition, out Quaternion cameraRotation)
{
#if !UNITY_EDITOR && UNITY_WSA
SpatialCoordinateSystem unityCoordinateSystem = Marshal.GetObjectForIUnknown(WorldManager.GetNativeISpatialCoordinateSystemPtr()) as SpatialCoordinateSystem;
if (unityCoordinateSystem == null)
{
Debug.LogError("Failed to get the native SpatialCoordinateSystem");
cameraPosition = default(Vector3);
cameraRotation = default(Quaternion);
return(false);
}
if (timeConversionCalendar == null)
{
timeConversionCalendar = new Calendar();
}
timeConversionCalendar.SetToNow();
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(timeConversionCalendar.GetDateTime());
if (perceptionTimestamp != null)
{
SpatialLocator locator = SpatialLocator.GetDefault();
if (locator != null)
{
SpatialLocation headPose = locator.TryLocateAtTimestamp(perceptionTimestamp, unityCoordinateSystem);
if (headPose != null)
{
var systemOrientation = headPose.Orientation;
var systemPostion = headPose.Position;
// Convert the orientation and position from Windows to Unity coordinate spaces
cameraRotation.x = -systemOrientation.X;
cameraRotation.y = -systemOrientation.Y;
cameraRotation.z = systemOrientation.Z;
cameraRotation.w = systemOrientation.W;
cameraPosition.x = systemPostion.X;
cameraPosition.y = systemPostion.Y;
cameraPosition.z = -systemPostion.Z;
return(true);
}
}
}
cameraPosition = default(Vector3);
cameraRotation = default(Quaternion);
return(false);
#else
cameraPosition = Camera.main.transform.position;
cameraRotation = Camera.main.transform.rotation;
return(true);
#endif
}
/// <summary>
/// Gets currently detected <see cref="SpatialInteractionSource"/>s. This list may
/// differ from the data provider's internal <see cref="activeControllers"/> registry.
/// </summary>
/// <returns>List of tracked <see cref="SpatialInteractionSource"/>s.</returns>
private IReadOnlyList <SpatialInteractionSourceState> GetDetectedSources()
{
if (WindowsMixedRealityUtilities.SpatialInteractionManager == null)
{
return(new List <SpatialInteractionSourceState>());
}
var perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
return(WindowsMixedRealityUtilities.SpatialInteractionManager.GetDetectedSourcesAtTimestamp(perceptionTimestamp));
}
// TODO: which is the smoothest experience? FixedUpdate for physics, or LateUpdate/Prerender for more accurate onscreen representation
// (or if we move away from physics, then see if LateUpdate is just better.
private void FixedUpdate()
{
#if WINDOWS_UWP
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(
DateTimeOffset.Now + TimeSpan.FromSeconds(Time.fixedDeltaTime * 3.0f));
if (!UpdateAtTime(perceptionTimestamp))
{
// prediction failed, fall back to current timestamp
UpdateAtTime(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
}
#endif
}
private void UpdateHands()
{
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(SpatialCoordinateSystem, jointIndices, jointPoses))
{
SpatialInteractionSourceHandedness handIndex = sourceState.Source.Handedness;
if (handIndex == SpatialInteractionSourceHandedness.Left)
{
ApplyTransforms(leftHandProxy, jointPoses);
}
else
{
ApplyTransforms(rightHandProxy, jointPoses);
}
}
}
}
private async Task GetController()
{
var access = await VibrationDevice.RequestAccessAsync();
if (access == VibrationAccessStatus.Allowed)
{
var mgr = SpatialInteractionManager.GetForCurrentView();
var calendar = new Calendar();
var timestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(calendar.GetDateTime());
controller = (from s in mgr.GetDetectedSourcesAtTimestamp(timestamp)
where s.Source.Id == ControllerID
select s.Source.Controller.SimpleHapticsController)
.FirstOrDefault();
expressions = new Dictionary <ushort, SimpleHapticsControllerFeedback>(5);
if (controller != null)
{
foreach (var fb in controller.SupportedFeedback)
{
if (fb.Waveform == KnownSimpleHapticsControllerWaveforms.BuzzContinuous)
{
buzz = fb;
}
else if (fb.Waveform == KnownSimpleHapticsControllerWaveforms.Click)
{
click = fb;
}
else if (fb.Waveform == KnownSimpleHapticsControllerWaveforms.Press)
{
press = fb;
}
else if (fb.Waveform == KnownSimpleHapticsControllerWaveforms.Release)
{
release = fb;
}
expressions[fb.Waveform] = fb;
}
}
}
}
// Update is called once per frame
void Update()
{
#if ENABLE_WINMD_SUPPORT
if (!_isReadyToRender)
{
return;
}
// The HolographicFrame has information that the app needs in order
// to update and render the current frame. The app begins each new
// frame by calling CreateNextFrame.
//HolographicFrame ^ holographicFrame = m_holographicSpace->CreateNextFrame();
// Get a prediction of where holographic cameras will be when this frame
// is presented.
//HolographicFramePrediction prediction = holographicFrame->CurrentPrediction;
IntPtr spatialCoordinateSystemPtr = WorldManager.GetNativeISpatialCoordinateSystemPtr();
SpatialCoordinateSystem unityWorldOrigin = Marshal.GetObjectForIUnknown(spatialCoordinateSystemPtr) as SpatialCoordinateSystem;
SpatialCoordinateSystem currentCoordinateSystem = unityWorldOrigin;
_isTrackingFaces = _faceTrackerProcessor.IsTrackingFaces();
if (_isTrackingFaces)
{
MediaFrameReference frame = _videoFrameProcessor.GetLatestFrame();
if (frame == null)
{
return;
}
var faces = _faceTrackerProcessor.GetLatestFaces();
ProcessFaces(faces, frame, currentCoordinateSystem);
TimeSpan currentTimeStamp = frame.SystemRelativeTime.Value.Duration();
if (currentTimeStamp > _previousFrameTimestamp)
{
// TODO: copy to texture
_previousFrameTimestamp = frame.SystemRelativeTime.Value.Duration();
}
}
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(currentCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
#endif
}
/// <inheritdoc />
public override void Update()
{
// Override gaze before base.Update() updates the controllers
if (mixedRealityGazeProviderHeadOverride != null && mixedRealityGazeProviderHeadOverride.UseHeadGazeOverride && WindowsMixedRealityUtilities.SpatialCoordinateSystem != null)
{
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
HeadPose head = pointerPose.Head;
if (head != null)
{
mixedRealityGazeProviderHeadOverride.OverrideHeadGaze(head.Position.ToUnityVector3(), head.ForwardDirection.ToUnityVector3());
}
}
}
base.Update();
}
public override void Update()
{
#if UNITY_WSA
if (WindowsMixedRealityUtilities.SpatialCoordinateSystem == null || typeof(SpatialPointerPose).GetProperty("Eyes") == null)
{
return;
}
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
var eyes = pointerPose.Eyes;
if (eyes != null)
{
InputSystem?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, eyes.IsCalibrationValid);
if (eyes.Gaze.HasValue)
{
Ray newGaze = new Ray(WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Origin), WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Direction));
if (SmoothEyeTracking)
{
newGaze = SmoothGaze(newGaze);
}
InputSystem?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, eyes.UpdateTimestamp.TargetTime.UtcDateTime);
}
}
}
#endif // UNITY_WSA
}
public override void Update()
{
#if WINDOWS_UWP
if (WindowsMixedRealityUtilities.SpatialCoordinateSystem == null || !WindowsApiChecker.UniversalApiContractV8_IsAvailable)
{
return;
}
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
var eyes = pointerPose.Eyes;
if ((eyes != null) && (eyes.Gaze.HasValue))
{
Ray newGaze = new Ray(WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Origin), WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Direction));
if (SmoothEyeTracking)
{
newGaze = SmoothGaze(newGaze);
}
InputSystem?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, eyes.UpdateTimestamp.TargetTime.UtcDateTime);
}
}
#endif // WINDOWS_UWP
}
public static IAsyncOperation <IRandomAccessStreamWithContentType> TryGetRenderableModelAsync(this InteractionSource interactionSource)
{
IAsyncOperation <IRandomAccessStreamWithContentType> returnValue = null;
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager.GetForCurrentView().GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSource.id))
{
//Build error
//returnValue = sourceState.Source.Controller.TryGetRenderableModelAsync();
}
}
}, true);
return(returnValue);
}
public static void StopHaptics(this InteractionSource interactionSource)
{
if (!WindowsApiChecker.UniversalApiContractV4_IsAvailable && !Application.isEditor)
{
return;
}
#if !UNITY_EDITOR
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager.GetForCurrentView().GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSource.id))
{
sourceState.Source.Controller.SimpleHapticsController.StopFeedback();
}
}
}, true);
#elif UNITY_EDITOR_WIN
StopHaptics(interactionSource.id);
#endif // !UNITY_EDITOR
}
/// <summary>
/// Gets the current native SpatialInteractionSourceState for this InteractionSource.
/// </summary>
/// <param name="interactionSource">This InteractionSource to search for via the native Windows APIs.</param>
/// <returns>The current native SpatialInteractionSourceState.</returns>
public static SpatialInteractionSourceState GetSpatialInteractionSourceState(this InteractionSource interactionSource)
{
IReadOnlyList <SpatialInteractionSourceState> sources = WindowsMixedRealityUtilities.SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.UtcNow));
for (var i = 0; i < sources?.Count; i++)
{
if (sources[i].Source.Id.Equals(interactionSource.id))
{
return(sources[i]);
}
}
return(null);
}
private IAsyncOperation <IRandomAccessStreamWithContentType> TryGetRenderableModelAsync(uint interactionSourceId)
{
IAsyncOperation <IRandomAccessStreamWithContentType> returnValue = null;
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
var sources = SpatialInteractionManager.GetForCurrentView()?.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (sources != null)
{
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceId))
{
returnValue = sourceState.Source.Controller.TryGetRenderableModelAsync();
}
}
}
}, true);
return(returnValue);
}
public override void StartHaptics(float normalizedIntensity, float durationInSeconds)
{
#if !UNITY_EDITOR && UNITY_2017_2_OR_NEWER && UNITY_WSA
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager
.GetForCurrentView()
.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper
.FromHistoricalTargetTime(DateTimeOffset
.Now));
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(_interactionSourceId))
{
SimpleHapticsController simpleHapticsController = sourceState
.Source
.Controller
.SimpleHapticsController;
foreach (SimpleHapticsControllerFeedback hapticsFeedback in simpleHapticsController.SupportedFeedback)
{
if (hapticsFeedback.Waveform.Equals(ContinuousBuzzWaveform))
{
if (durationInSeconds.Equals(float.MaxValue))
{
simpleHapticsController.SendHapticFeedback(hapticsFeedback, intensity);
}
else
{
simpleHapticsController.SendHapticFeedbackForDuration(
hapticsFeedback,
intensity,
TimeSpan.FromSeconds(durationInSeconds));
}
return;
}
}
}
}
}, true);
#endif
}
/// <inheritdoc/>
public override void Update()
{
Profiler.BeginSample("[MRTK] WindowsMixedRealityDeviceManager.Update");
base.Update();
#if (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
if (mixedRealityGazeProviderHeadOverride != null && mixedRealityGazeProviderHeadOverride.UseHeadGazeOverride)
{
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
HeadPose head = pointerPose.Head;
if (head != null)
{
mixedRealityGazeProviderHeadOverride.OverrideHeadGaze(head.Position.ToUnityVector3(), head.ForwardDirection.ToUnityVector3());
}
}
}
#endif // (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
UpdateInteractionManagerReading();
for (var i = 0; i < numInteractionManagerStates; i++)
{
// SourceDetected gets raised when a new controller is detected and, if previously present,
// when OnEnable is called. Do not create a new controller here.
var controller = GetOrAddController(interactionManagerStates[i].source, false);
if (controller != null)
{
controller.UpdateController(interactionManagerStates[i]);
}
}
LastInteractionManagerStateReading = interactionManagerStates;
Profiler.EndSample(); // Update
}
/// <summary>
/// Function which checks for new eye tracking data and is called periodically by the timer
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
private void CheckForEyeData(object source, ElapsedEventArgs e)
{
// Make sure the previous event isn't still running
if (System.Threading.Interlocked.CompareExchange(ref fetchDataTimerIsBusy, 1, 0) == 1)
{
//Debug.LogError("Previous event still running!");
return;
}
try {
#if (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
// Make sure we have the spatial coordinate system (which is cached every update) and the eyes API is available
if (currentSpatialCoordinateSystem == null || !EyesApiAvailable)
{
//Debug.Log("[UWPDataAccess] No currentSpatialCoordinateSystem or Eyes API not available!");
return;
}
// Try to get the new pointer data (which includes eye tracking)
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(currentSpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
// Check if we actually got any eye tracking data
var eyes = pointerPose.Eyes;
if (eyes != null)
{
// Unix time stamp from when the eye tracking data we got was acquired
long targetTimeUnix = eyes.UpdateTimestamp.TargetTime.ToUnixTimeMilliseconds();
// Check if we have new data
if (lastEyeDataTimestamp != targetTimeUnix)
{
// Save new time stamp
lastEyeDataTimestamp = targetTimeUnix;
// Save the information whether the calibration is valid
IsGazeCalibrationValid = eyes.IsCalibrationValid;
// If we have eye tracking data announce it in the event, otherwise simply announce Vector3.zero as origin and direction
if (eyes.Gaze.HasValue)
{
dataQueue.Enqueue(new GazeAPIData()
{
EyeDataTimestamp = targetTimeUnix,
EyeDataRelativeTimestamp = eyes.UpdateTimestamp.SystemRelativeTargetTime.TotalMilliseconds,
IsCalibrationValid = eyes.IsCalibrationValid,
GazeHasValue = eyes.Gaze.HasValue,
GazeOrigin = eyes.Gaze.Value.Origin.ToUnityVector3(),
GazeDirection = eyes.Gaze.Value.Direction.ToUnityVector3()
});
}
else
{
dataQueue.Enqueue(new GazeAPIData()
{
EyeDataTimestamp = targetTimeUnix,
EyeDataRelativeTimestamp = eyes.UpdateTimestamp.SystemRelativeTargetTime.TotalMilliseconds,
IsCalibrationValid = eyes.IsCalibrationValid,
GazeHasValue = eyes.Gaze.HasValue,
GazeOrigin = Vector3.zero,
GazeDirection = Vector3.zero
});
}
}
}
}
#else
// On all platforms which are not UWP print error
Debug.Log("[UWPDataAccess] Not on correct platform! Doing nothing!");
#endif
}
finally
{
fetchDataTimerIsBusy = 0;
}
}
/// <summary>
/// Tries to get an active SpatialInteractionSource with the corresponding handedness and input source type.
/// </summary>
/// <param name="handedness">The handedness of the source to get.</param>
/// <param name="inputSourceType">The input source type of the source to get.</param>
/// <returns>The input source or null if none could be found.</returns>
public static SpatialInteractionSource GetSpatialInteractionSource(Handedness handedness, InputSourceType inputSourceType)
{
SpatialInteractionSourceHandedness sourceHandedness;
switch (handedness)
{
default:
sourceHandedness = SpatialInteractionSourceHandedness.Unspecified;
break;
case Handedness.Left:
sourceHandedness = SpatialInteractionSourceHandedness.Left;
break;
case Handedness.Right:
sourceHandedness = SpatialInteractionSourceHandedness.Right;
break;
}
SpatialInteractionSourceKind sourceKind;
switch (inputSourceType)
{
default:
sourceKind = SpatialInteractionSourceKind.Other;
break;
case InputSourceType.Controller:
sourceKind = SpatialInteractionSourceKind.Controller;
break;
case InputSourceType.Hand:
sourceKind = SpatialInteractionSourceKind.Hand;
break;
}
System.Collections.Generic.IReadOnlyList <SpatialInteractionSourceState> sourceStates =
WindowsMixedRealityUtilities.SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(System.DateTimeOffset.UtcNow));
if (sourceStates == null)
{
return(null);
}
foreach (SpatialInteractionSourceState sourceState in sourceStates)
{
if (sourceState.Source.Handedness == sourceHandedness && sourceState.Source.Kind == sourceKind)
{
return(sourceState.Source);
}
}
return(null);
}
public static IAsyncOperation <IRandomAccessStreamWithContentType> TryGetRenderableModelAsync(this InteractionSource interactionSource)
{
IAsyncOperation <IRandomAccessStreamWithContentType> returnValue = null;
// GetForCurrentView and GetDetectedSourcesAtTimestamp were both introduced in the same Windows version.
// We need only check for one of them.
if (WindowsApiChecker.IsMethodAvailable(
"Windows.UI.Input.Spatial",
"SpatialInteractionManager",
"GetForCurrentView"))
{
IReadOnlyList <SpatialInteractionSourceState> sources = null;
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
sources = SpatialInteractionManager.GetForCurrentView()?.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
}, true);
for (var i = 0; i < sources?.Count; i++)
{
if (sources[i].Source.Id.Equals(interactionSource.id))
{
returnValue = sources[i].Source.Controller.TryGetRenderableModelAsync();
}
}
}
return(returnValue);
}
public override void Initialize(Chirality chirality)
{
#if !UNITY_EDITOR && UNITY_2017_2_OR_NEWER && UNITY_WSA
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager
.GetForCurrentView()
.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper
.FromHistoricalTargetTime(DateTimeOffset
.Now));
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Handedness == SpatialInteractionSourceHandedness.Left && chirality == Chirality.Left)
{
_interactionSourceId = sourceState.Source.Id;
}
if (sourceState.Source.Handedness == SpatialInteractionSourceHandedness.Right && chirality == Chirality.Right)
{
_interactionSourceId = sourceState.Source.Id;
}
}
}, true);
#endif
}
/// <inheritdoc />
public override void Update()
{
using (UpdatePerfMarker.Auto())
{
if (WindowsMixedRealityUtilities.SpatialCoordinateSystem == null || !eyesApiAvailable)
{
return;
}
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
var eyes = pointerPose.Eyes;
if (eyes != null)
{
Service?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, eyes.IsCalibrationValid);
if (eyes.Gaze.HasValue)
{
Ray newGaze = new Ray(eyes.Gaze.Value.Origin.ToUnityVector3(), eyes.Gaze.Value.Direction.ToUnityVector3());
if (SmoothEyeTracking)
{
newGaze = SmoothGaze(newGaze);
}
Service?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, eyes.UpdateTimestamp.TargetTime.UtcDateTime);
}
}
}
}
}
public override void StopHaptics()
{
#if !UNITY_EDITOR && UNITY_2017_2_OR_NEWER && UNITY_WSA
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager
.GetForCurrentView()
.GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper
.FromHistoricalTargetTime(DateTimeOffset
.Now));
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(_interactionSourceId))
{
sourceState.Source.Controller.SimpleHapticsController.StopFeedback();
}
}
}, true);
#endif
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
HandPose handPose = sourceState.TryGetHandPose();
#if WINDOWS_UWP
if (CoreServices.InputSystem.InputSystemProfile.HandTrackingProfile.EnableHandMeshVisualization)
{
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
if (handMeshObserver != null && handMeshTriangleIndices == null)
{
uint indexCount = handMeshObserver.TriangleIndexCount;
ushort[] indices = new ushort[indexCount];
handMeshObserver.GetTriangleIndices(indices);
handMeshTriangleIndices = new int[indexCount];
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
HandPose neutralPose = handMeshObserver.NeutralPose;
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(vertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralPoseVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
}
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
{
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Vector3 scale;
System.Numerics.Quaternion rotation;
System.Numerics.Vector3 translation;
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
handMeshVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
handMeshNormals[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Normal);
}
/// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
Vector3 unityPosition = WindowsMixedRealityUtilities.SystemVector3ToUnity(translation);
unityPosition = MixedRealityPlayspace.TransformPoint(unityPosition);
Quaternion unityRotation = WindowsMixedRealityUtilities.SystemQuaternionToUnity(rotation);
unityRotation = MixedRealityPlayspace.Rotation * unityRotation;
HandMeshInfo handMeshInfo = new HandMeshInfo
{
vertices = handMeshVertices,
normals = handMeshNormals,
triangles = handMeshTriangleIndices,
uvs = handMeshUVs,
position = unityPosition,
rotation = unityRotation
};
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
}
}
}
else
{
// if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// notify that hand mesh has been updated (cleared)
HandMeshInfo handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
}
#endif // WINDOWS_UWP
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
unityJointOrientations[i] = WindowsMixedRealityUtilities.SystemQuaternionToUnity(jointPoses[i].Orientation);
unityJointPositions[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(jointPoses[i].Position);
// We want the controller to follow the Playspace, so fold in the playspace transform here to
// put the controller pose into world space.
unityJointPositions[i] = MixedRealityPlayspace.TransformPoint(unityJointPositions[i]);
unityJointOrientations[i] = MixedRealityPlayspace.Rotation * unityJointOrientations[i];
if (jointIndices[i] == HandJointKind.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (!unityJointPoses.ContainsKey(handJoint))
{
unityJointPoses.Add(handJoint, new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]));
}
else
{
unityJointPoses[handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]);
}
}
CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, unityJointPoses);
}
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
public static IAsyncOperation <IRandomAccessStreamWithContentType> TryGetRenderableModelAsync(this InteractionSource interactionSource)
{
IAsyncOperation <IRandomAccessStreamWithContentType> returnValue = null;
if (WindowsApiChecker.UniversalApiContractV5_IsAvailable)
{
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager.GetForCurrentView().GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
for (var i = 0; i < sources.Count; i++)
{
if (sources[i].Source.Id.Equals(interactionSource.id))
{
returnValue = sources[i].Source.Controller.TryGetRenderableModelAsync();
}
}
}, true);
}
return(returnValue);
}
public static void StartHaptics(this InteractionSource interactionSource, float intensity, float durationInSeconds)
{
if (!WindowsApiChecker.UniversalApiContractV4_IsAvailable && !Application.isEditor)
{
return;
}
#if !UNITY_EDITOR
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager.GetForCurrentView().GetDetectedSourcesAtTimestamp(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSource.id))
{
SimpleHapticsController simpleHapticsController = sourceState.Source.Controller.SimpleHapticsController;
foreach (SimpleHapticsControllerFeedback hapticsFeedback in simpleHapticsController.SupportedFeedback)
{
if (hapticsFeedback.Waveform.Equals(ContinuousBuzzWaveform))
{
if (durationInSeconds.Equals(float.MaxValue))
{
simpleHapticsController.SendHapticFeedback(hapticsFeedback, intensity);
}
else
{
simpleHapticsController.SendHapticFeedbackForDuration(hapticsFeedback, intensity, TimeSpan.FromSeconds(durationInSeconds));
}
return;
}
}
}
}
}, true);
#elif UNITY_EDITOR_WIN
StartHaptics(interactionSource.id, intensity, durationInSeconds);
#endif // !UNITY_EDITOR
}
/// <inheritdoc />
public override void Update()
{
#if (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
if (WindowsMixedRealityUtilities.SpatialCoordinateSystem == null || !eyesApiAvailable)
{
return;
}
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
var eyes = pointerPose.Eyes;
if (eyes != null)
{
InputSystem?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, eyes.IsCalibrationValid);
if (eyes.Gaze.HasValue)
{
Ray newGaze = new Ray(eyes.Gaze.Value.Origin.ToUnityVector3(), eyes.Gaze.Value.Direction.ToUnityVector3());
if (SmoothEyeTracking)
{
newGaze = SmoothGaze(newGaze);
}
InputSystem?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, eyes.UpdateTimestamp.TargetTime.UtcDateTime);
}
}
}
#endif // (UNITY_WSA && DOTNETWINRT_PRESENT) || WINDOWS_UWP
}
/// <summary>
/// Updates the application state once per frame.
/// </summary>
public HolographicFrame Update()
{
// Before doing the timer update, there is some work to do per-frame
// to maintain holographic rendering. First, we will get information
// about the current frame.
// The HolographicFrame has information that the app needs in order
// to update and render the current frame. The app begins each new
// frame by calling CreateNextFrame.
var holographicFrame = holographicSpace.CreateNextFrame();
// Get a prediction of where holographic cameras will be when this frame
// is presented.
var prediction = holographicFrame.CurrentPrediction;
// Back buffers can change from frame to frame. Validate each buffer, and recreate
// resource views and depth buffers as needed.
deviceResources.EnsureCameraResources(holographicFrame, prediction);
// Next, we get a coordinate system from the attached frame of reference that is
// associated with the current frame. Later, this coordinate system is used for
// for creating the stereo view matrices when rendering the sample content.
var currentCoordinateSystem = referenceFrame.CoordinateSystem;
#if DRAW_SAMPLE_CONTENT
// Check for new input state since the last frame.
var pointerState = spatialInputHandler.CheckForInput();
if (null != pointerState)
{
// When a Pressed gesture is detected, the sample hologram will be repositioned
// two meters in front of the user.
spinningCubeRenderer.PositionHologram(
pointerState.TryGetPointerPose(currentCoordinateSystem)
);
}
#endif
timer.Tick(() =>
{
//
// TODO: Update scene objects.
//
// Put time-based updates here. By default this code will run once per frame,
// but if you change the StepTimer to use a fixed time step this code will
// run as many times as needed to get to the current step.
//
#if DRAW_SAMPLE_CONTENT
spinningCubeRenderer.Update(timer);
var perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
var spatialPointerPose = SpatialPointerPose.TryGetAtTimestamp(currentCoordinateSystem,
perceptionTimestamp);
_holoCursor.PositionHologram(spatialPointerPose);
_holoCursor.Update(spatialPointerPose);
#endif
});
// We complete the frame update by using information about our content positioning
// to set the focus point.
foreach (var cameraPose in prediction.CameraPoses)
{
#if DRAW_SAMPLE_CONTENT
// The HolographicCameraRenderingParameters class provides access to set
// the image stabilization parameters.
var renderingParameters = holographicFrame.GetRenderingParameters(cameraPose);
// SetFocusPoint informs the system about a specific point in your scene to
// prioritize for image stabilization. The focus point is set independently
// for each holographic camera.
// You should set the focus point near the content that the user is looking at.
// In this example, we put the focus point at the center of the sample hologram,
// since that is the only hologram available for the user to focus on.
// You can also set the relative velocity and facing of that content; the sample
// hologram is at a fixed point so we only need to indicate its position.
renderingParameters.SetFocusPoint(
currentCoordinateSystem,
spinningCubeRenderer.Position
);
#endif
}
// The holographic frame will be used to get up-to-date view and projection matrices and
// to present the swap chain.
return(holographicFrame);
}
