//called immediately after construction
public virtual void Initialize()
{
#if CVR_STEAMVR
CognitiveVR_Manager.PoseEvent += CognitiveVR_Manager_OnPoseEvent; //1.2
#endif
#if CVR_OCULUS
OVRManager.HMDMounted += OVRManager_HMDMounted;
OVRManager.HMDUnmounted += OVRManager_HMDUnmounted;
#endif
string hmdname = "none";
#if CVR_FOVE
hmdname = "fove";
#elif CVR_ARKIT
hmdname = "arkit";
#elif CVR_ARCORE
hmdname = "arcore";
#elif CVR_META
hmdname = "meta";
#elif UNITY_2017_2_OR_NEWER
string rawHMDName = UnityEngine.XR.XRDevice.model.ToLower();
#else
string rawHMDName = UnityEngine.VR.VRDevice.model.ToLower();
#endif
hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName);
#if CVR_TOBIIVR
_eyeTracker = Tobii.Research.Unity.VREyeTracker.Instance;
#endif
GazeCore.SetHMDType(hmdname);
cameraRoot = CameraTransform.root;
}
private void CognitiveVR_Manager_TickEvent()
{
RaycastHit hit = new RaycastHit();
Ray ray = new Ray(CameraTransform.position, GetWorldGazeDirection());
Vector3 gpsloc = new Vector3();
float compass = 0;
Vector3 floorPos = new Vector3();
GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos);
float hitDistance;
DynamicObject hitDynamic;
Vector3 hitWorld;
Vector2 hitcoord;
if (DynamicRaycast(ray.origin, ray.direction, CameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitcoord)) //hit dynamic
{
if (hitDynamic.ObjectId != null)
{
string ObjectId = hitDynamic.ObjectId.Id;
Vector3 LocalGaze = hitDynamic.transform.InverseTransformPointUnscaled(hitWorld);
hitDynamic.OnGaze(CognitiveVR_Preferences.S_SnapshotInterval);
var mediacomponent = hitDynamic.GetComponent <MediaComponent>();
if (mediacomponent != null)
{
var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0;
var mediauvs = hitcoord;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos);
}
else
{
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, ray.origin, CameraTransform.rotation, gpsloc, compass, floorPos);
}
return;
}
}
if (Physics.Raycast(ray, out hit, cam.farClipPlane))
{
Vector3 pos = CameraTransform.position;
Vector3 gazepoint = hit.point;
Quaternion rot = CameraTransform.rotation;
//hit world
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), gazepoint, pos, rot, gpsloc, compass, floorPos);
}
else //hit sky / farclip
{
Vector3 pos = CameraTransform.position;
Quaternion rot = CameraTransform.rotation;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos);
}
}
void OnHelperPostRender(Ray ray, Vector3 gazepoint)
{
//RaycastHit hit = new RaycastHit();
Vector3 gpsloc = new Vector3();
float compass = 0;
Vector3 floorPos = new Vector3();
GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos);
float hitDistance;
DynamicObject hitDynamic;
Vector3 hitWorld;
Vector2 hitcoord;
if (DynamicRaycast(ray.origin, ray.direction, CameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitcoord)) //hit dynamic
{
string ObjectId = hitDynamic.ObjectId.Id;
Vector3 LocalGaze = hitDynamic.transform.InverseTransformPointUnscaled(hitWorld);
hitDynamic.OnGaze(CognitiveVR_Preferences.S_SnapshotInterval);
var mediacomponent = hitDynamic.GetComponent <MediaComponent>();
if (mediacomponent != null)
{
var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0;
var mediauvs = hitcoord;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos);
}
else
{
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, floorPos);
}
Debug.DrawLine(CameraTransform.position, hitWorld, Color.magenta, 1);
return;
}
if (gazepoint.magnitude > CameraComponent.farClipPlane * 0.99f) //compare to farplane. skybox
{
Vector3 pos = CameraTransform.position;
Quaternion rot = CameraTransform.rotation;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos);
Debug.DrawRay(transform.position, transform.forward * CameraComponent.farClipPlane, Color.cyan, 1);
}
else
{
Vector3 pos = CameraTransform.position;
Quaternion rot = CameraTransform.rotation;
//hit world
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos + gazepoint, pos, rot, gpsloc, compass, floorPos);
Debug.DrawLine(pos, pos + gazepoint, Color.red, 1);
LastGazePoint = pos + gazepoint;
}
}
//called immediately after construction
public virtual void Initialize()
{
#if CVR_STEAMVR
CognitiveVR_Manager.PoseEvent += CognitiveVR_Manager_OnPoseEvent; //1.2
#endif
#if CVR_OCULUS
OVRManager.HMDMounted += OVRManager_HMDMounted;
OVRManager.HMDUnmounted += OVRManager_HMDUnmounted;
#endif
string hmdname = "none";
#if CVR_FOVE
hmdname = "fove";
#elif CVR_ARKIT
hmdname = "arkit";
#elif CVR_ARCORE
hmdname = "arcore";
#elif CVR_META
hmdname = "meta";
#elif UNITY_2019_1_OR_NEWER
string rawHMDName = UnityEngine.XR.InputDevices.GetDeviceAtXRNode(UnityEngine.XR.XRNode.Head).name;
//string rawHMDName = UnityEngine.XR.XRDevice.model.ToLower();
hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName);
#elif UNITY_2017_2_OR_NEWER
string rawHMDName = UnityEngine.XR.XRDevice.model.ToLower();
hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName);
#else
string rawHMDName = UnityEngine.VR.VRDevice.model.ToLower();
hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName);
#endif
#if CVR_AH
ah_calibrator = Calibrator.Instance;
#endif
#if CVR_PUPIL
gazeController = FindObjectOfType <PupilLabs.GazeController>();
if (gazeController != null)
{
gazeController.OnReceive3dGaze += ReceiveEyeData;
}
else
{
Debug.LogError("Pupil Labs GazeController is null!");
}
#endif
GazeCore.SetHMDType(hmdname);
cameraRoot = GameplayReferences.HMD.root;
}
private void CognitiveVR_Manager_TickEvent()
{
if (GameplayReferences.HMD == null)
{
return;
}
RaycastHit hit = new RaycastHit();
Ray ray = new Ray(GameplayReferences.HMD.position, GetWorldGazeDirection());
Vector3 gpsloc = new Vector3();
float compass = 0;
Vector3 floorPos = new Vector3();
GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos);
float hitDistance;
DynamicObject hitDynamic;
Vector3 hitWorld;
Vector3 hitLocal;
Vector2 hitcoord;
if (DynamicRaycast(ray.origin, ray.direction, GameplayReferences.HMDCameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitLocal, out hitcoord)) //hit dynamic
{
string ObjectId = hitDynamic.DataId;
var mediacomponent = hitDynamic.GetComponent <MediaComponent>();
if (mediacomponent != null)
{
var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0;
var mediauvs = hitcoord;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, GameplayReferences.HMD.position, GameplayReferences.HMD.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos);
}
else
{
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, ray.origin, GameplayReferences.HMD.rotation, gpsloc, compass, floorPos);
}
Debug.DrawLine(GameplayReferences.HMD.position, hitWorld, new Color(1, 0, 1, 0.5f), CognitiveVR_Preferences.Instance.SnapshotInterval);
Debug.DrawRay(hitWorld, Vector3.right, Color.red, 1);
Debug.DrawRay(hitWorld, Vector3.forward, Color.blue, 1);
Debug.DrawRay(hitWorld, Vector3.up, Color.green, 1);
return;
}
if (Physics.Raycast(ray, out hit, GameplayReferences.HMDCameraComponent.farClipPlane, CognitiveVR_Preferences.Instance.GazeLayerMask))
{
Vector3 pos = GameplayReferences.HMD.position;
Vector3 gazepoint = hit.point;
Quaternion rot = GameplayReferences.HMD.rotation;
//hit world
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), gazepoint, pos, rot, gpsloc, compass, floorPos);
Debug.DrawLine(pos, gazepoint, Color.red, CognitiveVR_Preferences.Instance.SnapshotInterval);
Debug.DrawRay(gazepoint, Vector3.right, Color.red, 10);
Debug.DrawRay(gazepoint, Vector3.forward, Color.blue, 10);
Debug.DrawRay(gazepoint, Vector3.up, Color.green, 10);
}
else //hit sky / farclip
{
Vector3 pos = GameplayReferences.HMD.position;
Quaternion rot = GameplayReferences.HMD.rotation;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos);
Debug.DrawRay(pos, GameplayReferences.HMD.forward * GameplayReferences.HMDCameraComponent.farClipPlane, Color.cyan, CognitiveVR_Preferences.Instance.SnapshotInterval);
}
}
private void OnHelperPostRender()
{
Ray ray = new Ray(CameraTransform.position, GetWorldGazeDirection());
Vector3 gpsloc = new Vector3();
float compass = 0;
Vector3 floorPos = new Vector3();
GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos);
//raycast for dynamic
float hitDistance;
DynamicObject hitDynamic;
Vector3 hitWorld;
string objectId = "";
Vector3 localGaze = Vector3.zero;
Vector2 hitcoord;
if (DynamicRaycast(ray.origin, ray.direction, CameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitcoord)) //hit dynamic
{
objectId = hitDynamic.ObjectId.Id;
localGaze = hitDynamic.transform.InverseTransformPointUnscaled(hitWorld);
}
//get depth world point
Vector3 point;
Vector3 gazedirection = GetWorldGazeDirection();
Vector2 screenGazePoint = GetViewportGazePoint();
bool hitworld = GetGazePoint(Resolution, Resolution, out point, CameraComponent.nearClipPlane, CameraComponent.farClipPlane, CameraTransform.forward, CameraTransform.position, screenGazePoint, gazedirection);
float depthDistance = Vector3.Distance(point, CameraTransform.position);
if (hitDistance > 0 && hitDistance < depthDistance) //hit a dynamic object closer than the scene depth
{
//TODO check for media
hitDynamic.OnGaze(CognitiveVR_Preferences.S_SnapshotInterval);
var mediacomponent = hitDynamic.GetComponent <MediaComponent>();
if (mediacomponent != null)
{
var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0;
var mediauvs = hitcoord;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), objectId, localGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos);
}
else
{
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), objectId, localGaze, ray.origin, CameraTransform.rotation, gpsloc, compass, floorPos);
}
Debug.DrawLine(CameraTransform.position, hitWorld, Color.magenta, 1);
}
else //didn't hit a dynamic, or hit one further away than the depth buffer
{
if (hitworld) //hit world
{
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), point, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, floorPos);
Debug.DrawLine(CameraTransform.position, point, Color.red, 1);
}
else //hit skybox
{
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), CameraTransform.position, CameraTransform.rotation, gpsloc, compass, floorPos);
Debug.DrawRay(CameraTransform.position, CameraTransform.forward * CameraComponent.farClipPlane, Color.cyan, 1);
}
}
}
void OnHelperPostRender(Ray ray, Vector3 gazeVector, Vector3 worldpos)
{
if (GameplayReferences.HMD == null)
{
return;
}
Vector3 gpsloc = new Vector3();
float compass = 0;
Vector3 floorPos = new Vector3();
GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos);
float hitDistance;
DynamicObject hitDynamic;
Vector3 hitWorld;
Vector3 hitLocal;
Vector2 hitcoord;
string ObjectId = "";
if (DynamicRaycast(ray.origin, ray.direction, GameplayReferences.HMDCameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitLocal, out hitcoord)) //hit dynamic
{
ObjectId = hitDynamic.DataId;
}
float depthDistance = Vector3.Distance(GameplayReferences.HMD.position, worldpos);
if (hitDistance > 0 && hitDistance < depthDistance)
{
var mediacomponent = hitDynamic.GetComponent <MediaComponent>();
if (mediacomponent != null)
{
var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0;
var mediauvs = hitcoord;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, GameplayReferences.HMD.position, GameplayReferences.HMD.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos);
}
else
{
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, GameplayReferences.HMD.position, GameplayReferences.HMD.rotation, gpsloc, compass, floorPos);
}
Debug.DrawLine(GameplayReferences.HMD.position, hitDynamic.transform.position + hitLocal, Color.magenta, 1);
Debug.DrawRay(worldpos, Vector3.right, Color.red, 1);
Debug.DrawRay(worldpos, Vector3.forward, Color.blue, 1);
Debug.DrawRay(worldpos, Vector3.up, Color.green, 1);
return;
}
if (gazeVector.magnitude > GameplayReferences.HMDCameraComponent.farClipPlane * 0.99f) //compare to farplane. skybox
{
Vector3 pos = GameplayReferences.HMD.position;
Quaternion rot = GameplayReferences.HMD.rotation;
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos);
Debug.DrawRay(pos, GameplayReferences.HMD.forward * GameplayReferences.HMDCameraComponent.farClipPlane, Color.cyan, 1);
}
else
{
Vector3 pos = GameplayReferences.HMD.position;
Quaternion rot = GameplayReferences.HMD.rotation;
//hit world
GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), worldpos, pos, rot, gpsloc, compass, floorPos);
//Debug.DrawLine(pos, pos + gazepoint, Color.red, 1);
Debug.DrawLine(ray.origin, worldpos, Color.yellow, 1);
Debug.DrawRay(worldpos, Vector3.right, Color.red, 1);
Debug.DrawRay(worldpos, Vector3.forward, Color.blue, 1);
Debug.DrawRay(worldpos, Vector3.up, Color.green, 1);
//Debug.Log("hit world");
LastGazePoint = worldpos;
}
}
