/// <summary>
/// On Update check if we need to invoke an action, process waiting data and if existing update the data access layer
/// </summary>
private void Update()
{
// Call the Unity update for the data access layer if it exists
dataAccessLayer?.UnityUpdate();
// Check if there are new actions to be executed and if yes execute them
if (actionQueue.Count > 0)
{
// Process all actions which are waiting to be processed
// Note: This isn't 100% thread save as we could end in a loop when there are new actions coming in faster than we are processing them.
// However, actions are added that rarely that we shouldn't run into issues.
while (actionQueue.TryDequeue(out Action action))
{
// Invoke the action from the queue
action.Invoke();
}
}
// Check if there is new data to process and if yes process it
if (dataQueue.Count > 0)
{
// Process all data which is waiting to be processed
// Note: This isn't 100% thread save as we could end in a loop when there is still new data coming in faster than we are processing it.
// However, data is added slowly enough that we shouldn't run into issues.
while (dataQueue.TryDequeue(out GazeAPIData gazeAPIData))
{
// Initialize the resulting data object with the API data
GazeData gazeData = new GazeData(gazeAPIData);
// Add the current frame time
gazeData.FrameTimestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
// If we have valid gaze data process it
if (gazeAPIData.GazeHasValue)
{
// Crate a gaze ray based on the gaze data
Ray gazeRay = new Ray(gazeAPIData.GazeOrigin, gazeAPIData.GazeDirection);
////
// The 3D gaze point is the actual position the wearer is looking at.
// As everything apart from the eye tracking layers is visible, we have to collide the gaze with every layer except the eye tracking layers
// Check if the gaze hits anything that isn't an AOI
gazeData.GazePointHit = Physics.Raycast(gazeRay, out RaycastHit hitInfo, Mathf.Infinity, notEyeTrackingLayerMask);
// If we hit something, write the hit info to the data
if (gazeData.GazePointHit)
{
// Write all info from the hit to the data object
gazeData.GazePoint = hitInfo.point;
gazeData.GazePointName = hitInfo.collider.name;
// Cache the transform of the game object which was hit
Transform hitTransform = hitInfo.collider.transform;
// Get the position of the hit in the local coordinates of the game object which was hit
gazeData.GazePointOnHit = hitTransform.InverseTransformPoint(hitInfo.point);
// Get the info about the object which was hit
gazeData.GazePointHitPosition = hitTransform.position;
gazeData.GazePointHitRotation = hitTransform.rotation.eulerAngles;
gazeData.GazePointHitScale = hitTransform.lossyScale;
// Update the position of the GazePoint visualization (only visible in the MRC view)
GazePointVis.transform.position = hitInfo.point;
// Get the position of the gaze point in the right and left eye if we have stereo rendering
if (mainCamera.stereoActiveEye != Camera.MonoOrStereoscopicEye.Mono)
{
gazeData.GazePointLeftDisplay = mainCamera.WorldToScreenPoint(hitInfo.point, Camera.MonoOrStereoscopicEye.Left);
gazeData.GazePointRightDisplay = mainCamera.WorldToScreenPoint(hitInfo.point, Camera.MonoOrStereoscopicEye.Right);
}
else
{
gazeData.GazePointLeftDisplay = null;
gazeData.GazePointRightDisplay = null;
}
// Also get the mono position (and always do this)
gazeData.GazePointMonoDisplay = mainCamera.WorldToScreenPoint(hitInfo.point, Camera.MonoOrStereoscopicEye.Mono);
// Get the position of the gaze point on the webcam image
gazeData.GazePointWebcam = webcamCamera.WorldToScreenPoint(hitInfo.point, Camera.MonoOrStereoscopicEye.Mono);
}
else
{
// Update the position of the GazePoint visualization (only visible in the MRC view)
GazePointVis.transform.position = Vector3.zero;
}
////
// To check for AOIs we do a separate ray cast on the AOI layer
// Check if the gaze hits a AOI
gazeData.GazePointAOIHit = Physics.Raycast(gazeRay, out hitInfo, Mathf.Infinity, eyeTrackingAOILayerMask);
// If we hit an AOI, write the hit info to data, otherwise simply leave it empty
if (gazeData.GazePointAOIHit)
{
// Write all info from the hit to the data object
gazeData.GazePointAOI = hitInfo.point;
gazeData.GazePointAOIName = hitInfo.collider.name;
// Cache the transform of the game object which was hit
Transform hitTransform = hitInfo.collider.transform;
// Get the position of the hit in the local coordinates of the game object which was hit
gazeData.GazePointAOIOnHit = hitTransform.InverseTransformPoint(hitInfo.point);
// Get the info about the object which was hit
gazeData.GazePointAOIHitPosition = hitTransform.position;
gazeData.GazePointAOIHitRotation = hitTransform.rotation.eulerAngles;
gazeData.GazePointAOIHitScale = hitTransform.lossyScale;
// Get the position of the gaze point on the web cam image
gazeData.GazePointAOIWebcam = webcamCamera.WorldToScreenPoint(hitInfo.point, Camera.MonoOrStereoscopicEye.Mono);
}
}
// Get the position of the game objects we want to log
// Create new data array
gazeData.positionInfos = new PositionInfo[PositionLoggedGameObjects.Count];
// Go through every game object and log its position
for (int i = 0; i < PositionLoggedGameObjects.Count; i++)
{
// Check if the game object still exists
gazeData.positionInfos[i].positionValid = PositionLoggedGameObjects[i] != null;
// If it still exists log its position
if (gazeData.positionInfos[i].positionValid)
{
// Name
gazeData.positionInfos[i].gameObjectName = PositionLoggedGameObjects[i].name;
// Position
Vector3 position = PositionLoggedGameObjects[i].transform.position;
gazeData.positionInfos[i].xPosition = position.x;
gazeData.positionInfos[i].yPosition = position.y;
gazeData.positionInfos[i].zPosition = position.z;
// Rotation
Vector3 rotation = PositionLoggedGameObjects[i].transform.rotation.eulerAngles;
gazeData.positionInfos[i].xRotation = rotation.x;
gazeData.positionInfos[i].yRotation = rotation.y;
gazeData.positionInfos[i].zRotation = rotation.z;
// Scale
Vector3 scale = PositionLoggedGameObjects[i].transform.lossyScale;
gazeData.positionInfos[i].xScale = scale.x;
gazeData.positionInfos[i].yScale = scale.y;
gazeData.positionInfos[i].zScale = scale.z;
}
}
// Invoke new data event
NewDataEvent?.Invoke(gazeData);
}
}
}
/// <summary>
/// Handle new eye tracking data by logging it
/// Note: We should be in the main Unity thread when receiving the event from the data provider, however logging should also work outside the main thread
/// </summary>
/// <param name="gazeData"></param>
private void NewDataHandler(GazeData gazeData)
{
// Start the resulting data string
StringBuilder logStringBuilder = new StringBuilder();
logStringBuilder.Append(gazeData.EyeDataTimestamp.ToString(ci));
logStringBuilder.Append(",");
// Note: Highest accuracy for the EyeDataRelativeTimestamp is 100ns so we don't loose information by outputting a fixed number of decimal places
logStringBuilder.Append(gazeData.EyeDataRelativeTimestamp.ToString("F4", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.FrameTimestamp.ToString(ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.IsCalibrationValid.ToString(ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazeHasValue.ToString(ci));
logStringBuilder.Append(",");
// If we have valid gaze data process it
if (gazeData.GazeHasValue)
{
// Append the info about the gaze to our log
logStringBuilder.Append(gazeData.GazeOrigin.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazeOrigin.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazeOrigin.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazeDirection.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazeDirection.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazeDirection.z.ToString("F5", ci));
logStringBuilder.Append(",");
// Did we hit any GameObject?
logStringBuilder.Append(gazeData.GazePointHit);
logStringBuilder.Append(",");
// If we did hit something on the gaze ray, write the hit info to the log, otherwise simply write NA
if (gazeData.GazePointHit)
{
logStringBuilder.Append(gazeData.GazePoint.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePoint.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePoint.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointName);
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointOnHit.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointOnHit.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointOnHit.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitPosition.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitPosition.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitPosition.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitRotation.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitRotation.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitRotation.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitScale.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitScale.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointHitScale.z.ToString("F5", ci));
logStringBuilder.Append(",");
if (gazeData.GazePointLeftDisplay.HasValue)
{
logStringBuilder.Append(gazeData.GazePointLeftDisplay.Value.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointLeftDisplay.Value.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointLeftDisplay.Value.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointRightDisplay.Value.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointRightDisplay.Value.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointRightDisplay.Value.z.ToString("F5", ci));
logStringBuilder.Append(",");
}
else
{
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,");
}
logStringBuilder.Append(gazeData.GazePointMonoDisplay.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointMonoDisplay.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointMonoDisplay.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointWebcam.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointWebcam.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointWebcam.z.ToString("F5", ci));
logStringBuilder.Append(",");
}
else
{
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,");
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,");
}
// Did we hit an AOI?
logStringBuilder.Append(gazeData.GazePointAOIHit);
logStringBuilder.Append(",");
// If we hit an AOI, write the hit info to the log, otherwise simply write NA
if (gazeData.GazePointAOIHit)
{
logStringBuilder.Append(gazeData.GazePointAOI.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOI.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOI.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIName);
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIOnHit.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIOnHit.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIOnHit.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitPosition.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitPosition.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitPosition.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitRotation.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitRotation.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitRotation.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitScale.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitScale.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIHitScale.z.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIWebcam.x.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIWebcam.y.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.GazePointAOIWebcam.z.ToString("F5", ci));
}
else
{
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,");
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA");
}
}
else
{
// No gaze data
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,");
// No gaze hit
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,");
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,");
// No AOI hit
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,NA,");
logStringBuilder.Append("NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA");
}
// If we are supposed to log the position of game objects, log them
// Note: We log the position even when we have no gaze data!
for (int i = 0; i < gazeData.positionInfos.Length; i++)
{
// Make sure the object does have a valid position
if (!gazeData.positionInfos[i].positionValid)
{
logStringBuilder.Append(",NA,NA,NA,NA,NA,NA,NA,NA,NA");
continue;
}
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].xPosition.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].yPosition.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].zPosition.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].xRotation.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].yRotation.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].zRotation.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].xScale.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].yScale.ToString("F5", ci));
logStringBuilder.Append(",");
logStringBuilder.Append(gazeData.positionInfos[i].zScale.ToString("F5", ci));
}
// Append the separator for the info to the output string
logStringBuilder.Append(",");
// If there is info we should log, log it
if (infoToLog.Count > 0)
{
// Add the missing infos to the string
while (infoToLog.TryDequeue(out string info))
{
// Append the string
logStringBuilder.Append(info);
// If there are more strings to append, add a separator between them
if (infoToLog.Count > 0)
{
logStringBuilder.Append(";");
}
}
}
// If there is nothing to log simply leave this column empty.
// This saves space and as we are at the end of the columns it doesn't mess up the other columns
// Write info to file
FileHandler.WriteData(logStringBuilder.ToString());
}