internal VideoCaptureSample(MediaFrameReference frameReference, SpatialCoordinateSystem worldOrigin)
{
if (frameReference == null)
{
throw new ArgumentNullException("frameReference.");
}
this.frameReference = frameReference;
this.worldOrigin = worldOrigin;
bitmap = frameReference.VideoMediaFrame.SoftwareBitmap;
}
/// <summary>
/// This returns the transform matrix at the time the photo was captured, if location data if available.
/// If it's not, that is probably an indication that the HoloLens is not tracking and its location is not known.
/// It could also mean the VideoCapture stream is not running.
/// If location data is unavailable then the camera to world matrix will be set to the identity matrix.
/// </summary>
/// <param name="matrix">The transform matrix used to convert between coordinate spaces.
/// The matrix will have to be converted to a Unity matrix before it can be used by methods in the UnityEngine namespace.
/// See https://forum.unity3d.com/threads/locatable-camera-in-unity.398803/ for details.</param>
public bool TryGetCameraToWorldMatrix(out float[] outMatrix)
{
if (frameReference.Properties.ContainsKey(viewTransformGuid) == false)
{
outMatrix = GetIdentityMatrixFloatArray();
return(false);
}
if (worldOrigin == null)
{
outMatrix = GetIdentityMatrixFloatArray();
return(false);
}
Matrix4x4 cameraViewTransform = ConvertByteArrayToMatrix4x4(frameReference.Properties[viewTransformGuid] as byte[]);
if (cameraViewTransform == null)
{
outMatrix = GetIdentityMatrixFloatArray();
return(false);
}
SpatialCoordinateSystem cameraCoordinateSystem = frameReference.Properties[cameraCoordinateSystemGuid] as SpatialCoordinateSystem;
if (cameraCoordinateSystem == null)
{
outMatrix = GetIdentityMatrixFloatArray();
return(false);
}
Matrix4x4?cameraCoordsToUnityCoordsMatrix = cameraCoordinateSystem.TryGetTransformTo(worldOrigin);
if (cameraCoordsToUnityCoordsMatrix == null)
{
outMatrix = GetIdentityMatrixFloatArray();
return(false);
}
Matrix4x4 worldToViewInCameraCoordsMatrix;
Matrix4x4.Invert(cameraViewTransform, out worldToViewInCameraCoordsMatrix);
Matrix4x4 worldToViewInUnityCoordsMatrix = Matrix4x4.Multiply(cameraCoordsToUnityCoordsMatrix.Value, worldToViewInCameraCoordsMatrix);
Matrix4x4 viewToWorldInCameraCoordsMatrix = Matrix4x4.Transpose(worldToViewInUnityCoordsMatrix);
viewToWorldInCameraCoordsMatrix.M31 *= -1f;
viewToWorldInCameraCoordsMatrix.M32 *= -1f;
viewToWorldInCameraCoordsMatrix.M33 *= -1f;
viewToWorldInCameraCoordsMatrix.M34 *= -1f;
outMatrix = ConvertMatrixToFloatArray(viewToWorldInCameraCoordsMatrix);
return(true);
}
// Use this for initialization
void Start()
{
#if WINDOWS_UWP
if (CoordinateSystem == null)
{
CoordinateSystem = Windows.Perception.Spatial.Preview.SpatialGraphInteropPreview.CreateCoordinateSystemForNode(id);
if (CoordinateSystem == null)
{
Debug.Log("Id= " + id + " Failed to acquire coordinate system");
}
}
#endif
}
private void Start()
{
#if UNITY_EDITOR || UNITY_WSA
UnityEngine.XR.WSA.WorldManager.OnPositionalLocatorStateChanged += WorldManager_OnPositionalLocatorStateChanged;
CurrentState = UnityEngine.XR.WSA.WorldManager.state;
#if WINDOWS_UWP
if (CoordinateSystem == null)
{
CoordinateSystem = Windows.Perception.Spatial.Preview.SpatialGraphInteropPreview.CreateCoordinateSystemForNode(id);
}
#endif // WINDOWS_UWP
#endif // UNITY_EDITOR || UNITY_WSA
}
private static SpatialCoordinateSystem RetrieveWorldOriginFromPointer(IntPtr worldOriginPtr)
{
if (worldOriginPtr == IntPtr.Zero)
{
throw new ArgumentException("World origin pointer is zero");
}
SpatialCoordinateSystem spatialCoordinateSystem = Marshal.GetObjectForIUnknown(worldOriginPtr) as SpatialCoordinateSystem;
if (spatialCoordinateSystem == null)
{
throw new InvalidCastException("Failed to retrieve world origin from pointer");
}
return(spatialCoordinateSystem);
}
public Matrix4x4 GetWorldToCameraMatrix(SpatialCoordinateSystem originCoordinateSystem)
{
lock (TransformLock)
{
Forward = new Vector4(-Vector3.UnitZ, 0.0f);
if (CoordinateSystem == null)
{
return(Matrix4x4.Identity);
}
var transform = originCoordinateSystem.TryGetTransformTo(CoordinateSystem) ?? Matrix4x4.Identity;
Matrix4x4.Invert(transform * ViewMatrix, out var inverseMatrix);
Forward = Vector4.Transform(Forward, inverseMatrix);
return(transform * ViewMatrix * ProjectionMatrix);
}
}
/// <summary>
/// use node 0 as the relative reference point for y
/// </summary>
/// <param name="referenceFrameCoordinateSystem"></param>
/// <returns></returns>
Entity GetReferenceEntitySpatialCoordinateSystem(SpatialCoordinateSystem referenceFrameCoordinateSystem)
{
if (nodes.Count == 0)
{
return(null);
}
var entity = new Entity($"base_node");
entity.Node = nodes[0];
entity.Renderer = nodeRenderer;
entity.UpdateTransform(referenceFrameCoordinateSystem);
return(entity);
}
public HoloLensCamera(CaptureMode captureMode, PixelFormat pixelFormat = PixelFormat.BGRA8)
{
desiredPixelFormat = pixelFormat;
CameraType = CameraType.Invalid;
CaptureMode = captureMode;
#if CAN_USE_UNITY_TYPES && UNITY_WSA && CAN_USE_UWP_TYPES
IntPtr coordinateSystemPtr;
// this must be done from the main thread, so done in
coordinateSystemPtr = UnityEngine.XR.WSA.WorldManager.GetNativeISpatialCoordinateSystemPtr();
if (coordinateSystemPtr != null)
{
rootCoordinateSystem = WinRTExtensions.GetSpatialCoordinateSystem(coordinateSystemPtr);
}
#endif
}
/// <summary>
/// We position the entities relative to the first node to keep the entities
/// positioned consistent
/// </summary>
/// <param name="referenceFrameCoordinateSystem"></param>
/// <returns></returns>
Entity GetRootEntity(SpatialCoordinateSystem referenceFrameCoordinateSystem)
{
if (nodes.Count == 0)
{
return(null);
}
var entity = new Entity($"node_root");
entity.Node = nodes[0];
entity.Renderer = nodeRenderer;
entity.Position = new Vector3(0, EntityOffsetY, 0);
entity.UpdateTransform(referenceFrameCoordinateSystem);
return(entity);
}
public void StartPullCameraFrames()
{
System.Diagnostics.Debug.WriteLine("StartPullCameraFrames");
Task.Run(async() =>
{
var ModelHelper = new ONNXModelHelper(UnityApp);
System.Diagnostics.Debug.WriteLine("model inited");
for (; ;) // Forever = While the app runs
{
FramesCaptured++;
await Task.Delay(PredictionFrequency);
using (var frameReference = CameraFrameReader.TryAcquireLatestFrame())
using (var videoFrame = frameReference?.VideoMediaFrame?.GetVideoFrame())
{
if (videoFrame == null)
{
System.Diagnostics.Debug.WriteLine("frame is null");
continue; //ignoring frame
}
if (videoFrame.Direct3DSurface == null)
{
System.Diagnostics.Debug.WriteLine("d3d surface is null");
videoFrame.Dispose();
continue; //ignoring frame
}
try
{
System.Diagnostics.Debug.WriteLine("trying to evaluate");
SpatialCoordinateSystem worldCoordinateSystem = m_referenceFrame.CoordinateSystem;
Matrix4x4 cameraToWorld = (Matrix4x4)frameReference.CoordinateSystem.TryGetTransformTo(worldCoordinateSystem);
CameraIntrinsics cameraIntrinsics = frameReference.VideoMediaFrame.CameraIntrinsics;
DepthMediaFrame depthFrame = frameReference.VideoMediaFrame.DepthMediaFrame;
await ModelHelper.EvaluateVideoFrameAsync(videoFrame, frameReference.VideoMediaFrame, worldCoordinateSystem, frameReference.CoordinateSystem).ConfigureAwait(false);
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex.Message);
}
finally
{
}
}
}
});
}
public SpatialSurfaceRenderer(DeviceResources deviceResources, SpatialCoordinateSystem spatialCoordinateSystem)
{
_deviceResources = deviceResources;
_spatialCoordinateSystem = spatialCoordinateSystem;
CheckAccess();
var desc = RasterizerStateDescription.Default();
desc.FillMode = FillMode.Wireframe;
_state = new RasterizerState(deviceResources.D3DDevice, desc);
SurfaceMeshList.CoordinateSystem = _spatialCoordinateSystem;
_surfaceMeshList.DirectXDevice = deviceResources.D3DDevice;
CreateDeviceDependentResourcesAsync();
}
// Update is called once per frame
void Update()
{
#if ENABLE_WINMD_SUPPORT
if (!_isReadyToRender)
{
return;
}
// The HolographicFrame has information that the app needs in order
// to update and render the current frame. The app begins each new
// frame by calling CreateNextFrame.
//HolographicFrame ^ holographicFrame = m_holographicSpace->CreateNextFrame();
// Get a prediction of where holographic cameras will be when this frame
// is presented.
//HolographicFramePrediction prediction = holographicFrame->CurrentPrediction;
IntPtr spatialCoordinateSystemPtr = WorldManager.GetNativeISpatialCoordinateSystemPtr();
SpatialCoordinateSystem unityWorldOrigin = Marshal.GetObjectForIUnknown(spatialCoordinateSystemPtr) as SpatialCoordinateSystem;
SpatialCoordinateSystem currentCoordinateSystem = unityWorldOrigin;
_isTrackingFaces = _faceTrackerProcessor.IsTrackingFaces();
if (_isTrackingFaces)
{
MediaFrameReference frame = _videoFrameProcessor.GetLatestFrame();
if (frame == null)
{
return;
}
var faces = _faceTrackerProcessor.GetLatestFaces();
ProcessFaces(faces, frame, currentCoordinateSystem);
TimeSpan currentTimeStamp = frame.SystemRelativeTime.Value.Duration();
if (currentTimeStamp > _previousFrameTimestamp)
{
// TODO: copy to texture
_previousFrameTimestamp = frame.SystemRelativeTime.Value.Duration();
}
}
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(currentCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
#endif
}
/// <summary>
/// Updates the application state once per frame.
/// </summary>
public HolographicFrame Update()
{
// Before doing the timer update, there is some work to do per-frame
// to maintain holographic rendering. First, we will get information
// about the current frame.
// The HolographicFrame has information that the app needs in order
// to update and render the current frame. The app begins each new
// frame by calling CreateNextFrame.
HolographicFrame holographicFrame = holographicSpace.CreateNextFrame();
// Get a prediction of where holographic cameras will be when this frame
// is presented.
HolographicFramePrediction prediction = holographicFrame.CurrentPrediction;
// Back buffers can change from frame to frame. Validate each buffer, and recreate
// resource views and depth buffers as needed.
deviceResources.EnsureCameraResources(holographicFrame, prediction);
// Next, we get a coordinate system from the attached frame of reference that is
// associated with the current frame. Later, this coordinate system is used for
// for creating the stereo view matrices when rendering the sample content.
SpatialCoordinateSystem currentCoordinateSystem = referenceFrame.CoordinateSystem;
timer.Tick(() =>
{
//
// TODO: Update scene objects.
//
// Put time-based updates here. By default this code will run once per frame,
// but if you change the StepTimer to use a fixed time step this code will
// run as many times as needed to get to the current step.
//
});
// We complete the frame update by using information about our content positioning
// to set the focus point.
foreach (var cameraPose in prediction.CameraPoses)
{
}
// The holographic frame will be used to get up-to-date view and projection matrices and
// to present the swap chain.
return(holographicFrame);
}
int RebuildTrailToTarget(SpatialCoordinateSystem referenceFrameCoordinateSystem, PerceptionTimestamp perceptionTimestamp, Node startNode, Node endNode, int lookAhead = 3)
{
Debug.WriteLine($"RebuildTrailToTarget {startNode.Name} -> {endNode.Name}");
entities.Clear();
var trail = new List <Node>();
if (startNode == endNode)
{
trail.Add(startNode);
}
else
{
BuildPath(endNode, startNode, trail, new List <Node>());
}
if (trail.Count == 0)
{
Debug.WriteLine($"Unable to find Path for startNode {startNode.Name} and {endNode.Name}");
return(-1);
}
Debug.WriteLine($"Creating trials {trail.ToArray()}");
var baseEntity = GetReferenceEntitySpatialCoordinateSystem(referenceFrameCoordinateSystem);
for (var i = 0; i < Math.Min(trail.Count, lookAhead); i++)
{
var node = trail[i];
var entity = new Entity($"node_{i}");
entity.Node = node;
entity.Renderer = nodeRenderer;
entity.UpdateTransform(referenceFrameCoordinateSystem);
// offset from baseEntity (to keep the y positions uniform and consistent)
var targetPosition = baseEntity.Transform.Translation;
entity.Position = new Vector3(0, (targetPosition - entity.Transform.Translation).Y, 0f);
entities.Add(entity);
}
return(1);
}
public IList <Node> FindClosestNodesWithSightedItem(SpatialCoordinateSystem referenceFrameCoordinateSystem, SpatialPointerPose pose, string sightingItem)
{
var filteredNodes = nodes.Where(node =>
{
return(node.Sightings.Any(sighting =>
{
return sighting.Tokens.Any(token => token.Equals(sightingItem, StringComparison.OrdinalIgnoreCase));
}));
});
if (filteredNodes != null)
{
return(filteredNodes.OrderBy(node =>
{
return node.TryGetDistance(referenceFrameCoordinateSystem, pose.Head.Position);
}).ToList());
}
return(null);
}
public async void Initialize(SpatialCoordinateSystem coordinateSystem)
{
CoordinateSystem = coordinateSystem;
var requestStatus = await SpatialSurfaceObserver.RequestAccessAsync();
if (requestStatus == SpatialPerceptionAccessStatus.Allowed)
{
SurfaceObserver = new SpatialSurfaceObserver();
var boundingBox = new SpatialBoundingBox()
{
Center = Vector3.Zero,
Extents = new Vector3(10.0f, 10.0f, 2.5f)
};
SurfaceObserver.SetBoundingVolume(SpatialBoundingVolume.FromBox(coordinateSystem, boundingBox));
await CreateDeviceDenpendantResources();
SurfaceObserver.ObservedSurfacesChanged += OnObservedSurfacesChanged;
Active = true;
}
}
void IRenderLoopHost.OnRenderLoop_PrepareRendering(EngineDevice device)
{
// Before doing the timer update, there is some work to do per-frame
// to maintain holographic rendering. First, we will get information
// about the current frame.
// The HolographicFrame has information that the app needs in order
// to update and render the current frame. The app begins each new
// frame by calling CreateNextFrame.
HolographicFrame holographicFrame = m_holoSpace.CreateNextFrame();
// Get a prediction of where holographic cameras will be when this frame
// is presented.
HolographicFramePrediction prediction = holographicFrame.CurrentPrediction;
// Next, we get a coordinate system from the attached frame of reference that is
// associated with the current frame. Later, this coordinate system is used for
// for creating the stereo view matrices when rendering the sample content.
SpatialCoordinateSystem currentCoordinateSystem = m_referenceFrame.CoordinateSystem;
}
internal async void setUpSpatialMapping(SpatialCoordinateSystem coordinateSystem)
{
if (!Windows.Foundation.Metadata.ApiInformation.IsApiContractPresent("Windows.Foundation.UniversalApiContract", 4) || SpatialSurfaceObserver.IsSupported())
{
SpatialPerceptionAccessStatus status = await SpatialSurfaceObserver.RequestAccessAsync();
if (status == SpatialPerceptionAccessStatus.Allowed)
{
SpatialSurfaceObserver observer = new SpatialSurfaceObserver();
SpatialBoundingBox boundingBox = new SpatialBoundingBox()
{
Center = new System.Numerics.Vector3(0, 0, 0),
Extents = new System.Numerics.Vector3(40, 40, 5)
};
SpatialBoundingVolume bounds = SpatialBoundingVolume.FromBox(coordinateSystem, boundingBox);
observer.SetBoundingVolume(bounds);
observer.ObservedSurfacesChanged += new TypedEventHandler <SpatialSurfaceObserver, object>(ClockworkSocket.SurfacesChanged);
}
}
}
void CreateEntitiesForAllNodes(SpatialCoordinateSystem referenceFrameCoordinateSystem)
{
entities.Clear();
var rootEntity = GetRootEntity(referenceFrameCoordinateSystem);
var i = 0;
foreach (var node in this.nodes)
{
var entity = new Entity($"node_{i}");
entity.Node = node;
entity.Renderer = nodeRenderer;
entity.UpdateTransform(referenceFrameCoordinateSystem);
var targetPosition = rootEntity.Transform.Translation;
entity.Position = new Vector3(0, (targetPosition - entity.Transform.Translation).Y, 0f);
entities.Add(entity);
i += 1;
}
}
internal VideoCaptureSample(MediaFrameReference frameReference, SpatialCoordinateSystem worldOrigin)
{
if (frameReference == null)
{
throw new ArgumentNullException("frameReference.");
}
this.frameReference = frameReference;
this.worldOrigin = worldOrigin;
// When Windows.Media.Devices.Core.CameraIntrinsics is out of prerelease, use this instead
//cameraIntrinsics = new CameraIntrinsics(frameReference.VideoMediaFrame.CameraIntrinsics);
byte[] rawIntrinsics = frameReference.Properties[cameraIntrinsicsGuid] as byte[];
float[] intrinsicArray = ConvertByteArrayToFloatArray(rawIntrinsics);
cameraIntrinsics = new CameraIntrinsics(intrinsicArray);
bitmap = frameReference.VideoMediaFrame.SoftwareBitmap;
FrameWidth = bitmap.PixelWidth;
FrameHeight = bitmap.PixelHeight;
}
public void UpdateTransform(SpatialCoordinateSystem referenceFrameCoordinateSystem)
{
if (!Enabled)
{
return;
}
var trans = Node.GetTransform(referenceFrameCoordinateSystem);
if (trans.HasValue)
{
Matrix4x4 modelTranslation = Matrix4x4.CreateTranslation(Position);
Matrix4x4 modelRotation = Matrix4x4.CreateFromYawPitchRoll(
DegreeToRadian(EulerAngles.Y),
DegreeToRadian(EulerAngles.X),
DegreeToRadian(EulerAngles.Z));
Matrix4x4 modelScale = Matrix4x4.CreateScale(Scale);
transform = (modelScale * modelRotation * modelTranslation) * trans.Value;
}
}
/// <summary>
/// Cache the coordinate system for the QR code's spatial node, and the root.
/// </summary>
/// <returns></returns>
private bool CheckCoordinateSystem()
{
#if WINDOWS_UWP
if (coordinateSystem == null)
{
SimpleConsole.AddLine(trace, $"Creating coord for {spatialNodeId}");
coordinateSystem = global::Windows.Perception.Spatial.Preview.SpatialGraphInteropPreview.CreateCoordinateSystemForNode(SpatialNodeId);
}
if (rootCoordinateSystem == null)
{
rootCoordinateSystem = System.Runtime.InteropServices.Marshal.GetObjectForIUnknown(
UnityEngine.XR.WSA.WorldManager.GetNativeISpatialCoordinateSystemPtr()
) as SpatialCoordinateSystem;
SimpleConsole.AddLine(trace, $"Getting root coordinate system {(rootCoordinateSystem == null ? "null" : "succeeded")}");
}
return(coordinateSystem != null);
#else // WINDOWS_UWP
return(false);
#endif // WINDOWS_UWP
}
public void Update(PerceptionTimestamp timeStamp, SpatialCoordinateSystem coordinateSystem)
{
var states = interactionManager.GetDetectedSourcesAtTimestamp(timeStamp);
foreach (SpatialInteractionSourceState state in states)
{
if (state.Source.Handedness == hand)
{
SpatialInteractionSourceLocation location = state.Properties.TryGetLocation(coordinateSystem);
if (location != null)
{
SetSpatialInteractionSourceLocation(location);
}
previousState = currentState;
currentState = state;
internalState = previousState != null ? DeviceState.Valid : DeviceState.Invalid;
}
}
}
int RebuildTrailToTarget(SpatialCoordinateSystem referenceFrameCoordinateSystem, PerceptionTimestamp perceptionTimestamp, Node startNode, Node endNode,
int lookAhead = 100)
{
Debug.WriteLine($"RebuildTrailToTarget {startNode.Name} -> {endNode.Name}");
entities.Clear();
Stack <Node> trail = new Stack <Node>();
BuildPath(endNode, startNode, trail);
if (trail.Count == 0)
{
Debug.WriteLine($"Unable to find Path for startNode {startNode.Name} and {endNode.Name}");
return(-1);
}
var rootEntity = GetRootEntity(referenceFrameCoordinateSystem);
int i = 0;
while (i < lookAhead && trail.Count > 0)
{
var node = trail.Pop();
var entity = new Entity($"node_{i}");
entity.Node = node;
entity.Renderer = entity.Node == targetNode ? targetNodeRenderer : nodeRenderer;
entity.UpdateTransform(referenceFrameCoordinateSystem);
var targetPosition = rootEntity.Transform.Translation;
entity.Position = new Vector3(0, (targetPosition - entity.Transform.Translation).Y, 0f);
entities.Add(entity);
i += 1;
}
return(1);
}
// Get transform matrices from the MediaFrameReference
public static Tuple <Matrix4x4, Matrix4x4, Matrix4x4> GetTransforms(MediaFrameReference colorFrameRef, SpatialCoordinateSystem unityWorldCoordinateSystem)
{
SpatialCoordinateSystem spatialCoordinateSystem = null;
Matrix4x4 projectionTransform = Matrix4x4.Identity; // intrinsics; does not change
Matrix4x4 viewTransform = Matrix4x4.Identity; // extrinsics; changes per frame
// TODO: Unity has CameraToWorldMatrix provided by PhotoCaptureFrame class... Cam space -> world space
// also has worldToCameraMatrix, can it replace cameraCoordinateSystem transforms?
// UnityEngine.Matrix4x4 camToWorld = UnityEngine.Camera.main.cameraToWorldMatrix;
object value;
if (colorFrameRef.Properties.TryGetValue(MFSampleExtension_Spatial_CameraCoordinateSystem, out value))
{
spatialCoordinateSystem = value as SpatialCoordinateSystem;
}
if (colorFrameRef.Properties.TryGetValue(MFSampleExtension_Spatial_CameraProjectionTransform, out value))
{
projectionTransform = ByteArrayToMatrix(value as byte[]);
}
if (colorFrameRef.Properties.TryGetValue(MFSampleExtension_Spatial_CameraViewTransform, out value))
{
viewTransform = ByteArrayToMatrix(value as byte[]);
}
// Transform: Camera Coord System -> Unity world coord
// See https://github.com/Microsoft/MixedRealityToolkit-Unity/blob/96cc9ab8998280edcd6871f41e89584030ee4f26/Assets/HoloToolkit-Preview/QRTracker/Scripts/SpatialGraphCoordinateSystem.cs#L94
var cameraRGBToWorldTransform = spatialCoordinateSystem.TryGetTransformTo(unityWorldCoordinateSystem);
if (cameraRGBToWorldTransform == null)
{
return(null);
}
Matrix4x4 frameToOrigin = cameraRGBToWorldTransform.Value;
return(Tuple.Create(frameToOrigin, projectionTransform, viewTransform));
}
private CameraExtrinsics GetExtrinsics(SpatialCoordinateSystem frameCoordinateSystem)
{
if (frameCoordinateSystem == null)
{
return(null);
}
CameraExtrinsics extrinsics = null;
if (rootCoordinateSystem == null)
{
return(null);
}
System.Numerics.Matrix4x4?worldMatrix = frameCoordinateSystem.TryGetTransformTo(rootCoordinateSystem);
if (worldMatrix.HasValue)
{
WindowsVector3 position;
WindowsVector3 scale;
WindowsQuaternion rotation;
WindowsMatrix4x4.Decompose(worldMatrix.Value, out scale, out rotation, out position);
WindowsVector3 forward = WindowsVector3.Transform(-WindowsVector3.UnitZ, rotation);
WindowsVector3 up = WindowsVector3.Transform(WindowsVector3.UnitY, rotation);
Matrix4x4 unityWorldMatrix = Matrix4x4.TRS(WindowsVectorToUnityVector(position), Quaternion.LookRotation(WindowsVectorToUnityVector(forward), WindowsVectorToUnityVector(up)), Vector3.one);
extrinsics = new CameraExtrinsics()
{
ViewFromWorld = unityWorldMatrix
};
}
return(extrinsics);
}
void ProcessFrame(SpatialCoordinateSystem worldCoordinateSystem)
{
if (!IsInValidateStateToProcessFrame())
{
return;
}
// obtain the details of the last frame captured
FrameGrabber.Frame frame = frameGrabber.LastFrame;
if (frame.mediaFrameReference == null)
{
return;
}
MediaFrameReference mediaFrameReference = frame.mediaFrameReference;
SpatialCoordinateSystem cameraCoordinateSystem = mediaFrameReference.CoordinateSystem;
CameraIntrinsics cameraIntrinsics = mediaFrameReference.VideoMediaFrame.CameraIntrinsics;
Matrix4x4? cameraToWorld = cameraCoordinateSystem.TryGetTransformTo(worldCoordinateSystem);
if (!cameraToWorld.HasValue)
{
return;
}
// padding
float averageFaceWidthInMeters = 0.15f;
float pixelsPerMeterAlongX = cameraIntrinsics.FocalLength.X;
float averagePixelsForFaceAt1Meter = pixelsPerMeterAlongX * averageFaceWidthInMeters;
// Place the label 25cm above the center of the face.
Vector3 labelOffsetInWorldSpace = new Vector3(0.0f, 0.25f, 0.0f);
frameAnalyzer.AnalyzeFrame(frame.mediaFrameReference, (status, detectedPersons) =>
{
if(status > 0 && detectedPersons.Count > 0)
{
FrameAnalyzer.Bounds? bestRect = null;
Vector3 bestRectPositionInCameraSpace = Vector3.Zero;
float bestDotProduct = -1.0f;
FrameAnalyzer.DetectedPerson bestPerson = null;
foreach (var dp in detectedPersons)
{
Debug.WriteLine($"Detected person: {dp.ToString()}");
Point faceRectCenterPoint = new Point(
dp.bounds.left + dp.bounds.width /2,
dp.bounds.top + dp.bounds.height / 2
);
// Calculate the vector towards the face at 1 meter.
Vector2 centerOfFace = cameraIntrinsics.UnprojectAtUnitDepth(faceRectCenterPoint);
// Add the Z component and normalize.
Vector3 vectorTowardsFace = Vector3.Normalize(new Vector3(centerOfFace.X, centerOfFace.Y, -1.0f));
// Get the dot product between the vector towards the face and the gaze vector.
// The closer the dot product is to 1.0, the closer the face is to the middle of the video image.
float dotFaceWithGaze = Vector3.Dot(vectorTowardsFace, -Vector3.UnitZ);
// Pick the faceRect that best matches the users gaze.
if (dotFaceWithGaze > bestDotProduct)
{
// Estimate depth using the ratio of the current faceRect width with the average faceRect width at 1 meter.
float estimatedFaceDepth = averagePixelsForFaceAt1Meter / (float)dp.bounds.width;
// Scale the vector towards the face by the depth, and add an offset for the label.
Vector3 targetPositionInCameraSpace = vectorTowardsFace * estimatedFaceDepth;
bestDotProduct = dotFaceWithGaze;
bestRect = dp.bounds;
bestRectPositionInCameraSpace = targetPositionInCameraSpace;
bestPerson = dp;
}
}
if (bestRect.HasValue)
{
// Transform the cube from Camera space to World space.
Vector3 bestRectPositionInWorldspace = Vector3.Transform(bestRectPositionInCameraSpace, cameraToWorld.Value);
Vector3 labelPosition = bestRectPositionInWorldspace + labelOffsetInWorldSpace;
quadRenderer.TargetPosition = labelPosition;
textRenderer.RenderTextOffscreen($"{bestPerson.name}, {bestPerson.gender}, Age: {bestPerson.age}");
lastFaceDetectedTimestamp = Utils.GetCurrentUnixTimestampMillis();
}
}
});
}
/// <summary>
/// Updates the application state once per frame.
/// </summary>
public HolographicFrame Update()
{
// Before doing the timer update, there is some work to do per-frame
// to maintain holographic rendering. First, we will get information
// about the current frame.
// The HolographicFrame has information that the app needs in order
// to update and render the current frame. The app begins each new
// frame by calling CreateNextFrame.
HolographicFrame holographicFrame = holographicSpace.CreateNextFrame();
// Get a prediction of where holographic cameras will be when this frame
// is presented.
HolographicFramePrediction prediction = holographicFrame.CurrentPrediction;
// Back buffers can change from frame to frame. Validate each buffer, and recreate
// resource views and depth buffers as needed.
deviceResources.EnsureCameraResources(holographicFrame, prediction);
// Next, we get a coordinate system from the attached frame of reference that is
// associated with the current frame. Later, this coordinate system is used for
// for creating the stereo view matrices when rendering the sample content.
SpatialCoordinateSystem currentCoordinateSystem = referenceFrame.CoordinateSystem;
SpatialPointerPose pose = SpatialPointerPose.TryGetAtTimestamp(currentCoordinateSystem, prediction.Timestamp);
ProcessFrame(currentCoordinateSystem);
if (Utils.GetCurrentUnixTimestampMillis() - lastFaceDetectedTimestamp > faceTimeThreshold)
{
if(pose != null)
{
var headPosition = pose.Head.Position;
var headForward = pose.Head.ForwardDirection;
quadRenderer.TargetPosition = headPosition + (2.0f * headForward);
}
textRenderer.RenderTextOffscreen("No faces detected");
}
timer.Tick(() =>
{
//
// TODO: Update scene objects.
//
// Put time-based updates here. By default this code will run once per frame,
// but if you change the StepTimer to use a fixed time step this code will
// run as many times as needed to get to the current step.
//
quadRenderer.Update(pose, timer);
});
// We complete the frame update by using information about our content positioning
// to set the focus point.
foreach (var cameraPose in prediction.CameraPoses)
{
// The HolographicCameraRenderingParameters class provides access to set
// the image stabilization parameters.
HolographicCameraRenderingParameters renderingParameters = holographicFrame.GetRenderingParameters(cameraPose);
// SetFocusPoint informs the system about a specific point in your scene to
// prioritize for image stabilization. The focus point is set independently
// for each holographic camera.
// You should set the focus point near the content that the user is looking at.
// In this example, we put the focus point at the center of the sample hologram,
// since that is the only hologram available for the user to focus on.
// You can also set the relative velocity and facing of that content; the sample
// hologram is at a fixed point so we only need to indicate its position.
if(Utils.GetCurrentUnixTimestampMillis() - lastFaceDetectedTimestamp <= faceTimeThreshold)
{
renderingParameters.SetFocusPoint(
currentCoordinateSystem,
quadRenderer.Position,
quadRenderer.Forward,
quadRenderer.Velocity
);
}
}
// The holographic frame will be used to get up-to-date view and projection matrices and
// to present the swap chain.
return holographicFrame;
}
private static extern void GetSpatialCoordinateSystem(IntPtr nativePtr, out SpatialCoordinateSystem coordinateSystem);
/// <summary>
/// Updates the constant buffer for the display with view and projection
/// matrices for the current frame.
/// </summary>
public void UpdateViewProjectionBuffer(
DeviceResources deviceResources,
HolographicCameraPose cameraPose,
SpatialCoordinateSystem coordinateSystem
)
{
// The system changes the viewport on a per-frame basis for system optimizations.
_d3DViewport.X = (float)cameraPose.Viewport.Left;
_d3DViewport.Y = (float)cameraPose.Viewport.Top;
_d3DViewport.Width = (float)cameraPose.Viewport.Width;
_d3DViewport.Height = (float)cameraPose.Viewport.Height;
_d3DViewport.MinDepth = 0;
_d3DViewport.MaxDepth = 1;
// The projection transform for each frame is provided by the HolographicCameraPose.
var cameraProjectionTransform = cameraPose.ProjectionTransform;
// Get a container object with the view and projection matrices for the given
// pose in the given coordinate system.
var viewTransformContainer = cameraPose.TryGetViewTransform(coordinateSystem);
// If TryGetViewTransform returns null, that means the pose and coordinate system
// cannot be understood relative to one another; content cannot be rendered in this
// coordinate system for the duration of the current frame.
// This usually means that positional tracking is not active for the current frame, in
// which case it is possible to use a SpatialLocatorAttachedFrameOfReference to render
// content that is not world-locked instead.
var viewProjectionConstantBufferData = new ViewProjectionConstantBuffer();
var viewTransformAcquired = viewTransformContainer.HasValue;
if (viewTransformAcquired)
{
// Otherwise, the set of view transforms can be retrieved.
var viewCoordinateSystemTransform = viewTransformContainer.Value;
// Update the view matrices. Holographic cameras (such as Microsoft HoloLens) are
// constantly moving relative to the world. The view matrices need to be updated
// every frame.
viewProjectionConstantBufferData.ViewProjectionLeft = Matrix4x4.Transpose(
viewCoordinateSystemTransform.Left * cameraProjectionTransform.Left
);
viewProjectionConstantBufferData.ViewProjectionRight = Matrix4x4.Transpose(
viewCoordinateSystemTransform.Right * cameraProjectionTransform.Right
);
}
// Use the D3D device context to update Direct3D device-based resources.
var context = deviceResources.D3DDeviceContext;
// Loading is asynchronous. Resources must be created before they can be updated.
if (context == null || _viewProjectionConstantBuffer == null || !viewTransformAcquired)
{
_framePending = false;
}
else
{
// Update the view and projection matrices.
context.UpdateSubresource(ref viewProjectionConstantBufferData, _viewProjectionConstantBuffer);
_framePending = true;
}
}
