//called immediately after construction public virtual void Initialize() { #if CVR_STEAMVR CognitiveVR_Manager.PoseEvent += CognitiveVR_Manager_OnPoseEvent; //1.2 #endif #if CVR_OCULUS OVRManager.HMDMounted += OVRManager_HMDMounted; OVRManager.HMDUnmounted += OVRManager_HMDUnmounted; #endif string hmdname = "none"; #if CVR_FOVE hmdname = "fove"; #elif CVR_ARKIT hmdname = "arkit"; #elif CVR_ARCORE hmdname = "arcore"; #elif CVR_META hmdname = "meta"; #elif UNITY_2017_2_OR_NEWER string rawHMDName = UnityEngine.XR.XRDevice.model.ToLower(); #else string rawHMDName = UnityEngine.VR.VRDevice.model.ToLower(); #endif hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName); #if CVR_TOBIIVR _eyeTracker = Tobii.Research.Unity.VREyeTracker.Instance; #endif GazeCore.SetHMDType(hmdname); cameraRoot = CameraTransform.root; }
private void CognitiveVR_Manager_TickEvent() { RaycastHit hit = new RaycastHit(); Ray ray = new Ray(CameraTransform.position, GetWorldGazeDirection()); Vector3 gpsloc = new Vector3(); float compass = 0; Vector3 floorPos = new Vector3(); GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos); float hitDistance; DynamicObject hitDynamic; Vector3 hitWorld; Vector2 hitcoord; if (DynamicRaycast(ray.origin, ray.direction, CameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitcoord)) //hit dynamic { if (hitDynamic.ObjectId != null) { string ObjectId = hitDynamic.ObjectId.Id; Vector3 LocalGaze = hitDynamic.transform.InverseTransformPointUnscaled(hitWorld); hitDynamic.OnGaze(CognitiveVR_Preferences.S_SnapshotInterval); var mediacomponent = hitDynamic.GetComponent <MediaComponent>(); if (mediacomponent != null) { var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0; var mediauvs = hitcoord; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos); } else { GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, ray.origin, CameraTransform.rotation, gpsloc, compass, floorPos); } return; } } if (Physics.Raycast(ray, out hit, cam.farClipPlane)) { Vector3 pos = CameraTransform.position; Vector3 gazepoint = hit.point; Quaternion rot = CameraTransform.rotation; //hit world GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), gazepoint, pos, rot, gpsloc, compass, floorPos); } else //hit sky / farclip { Vector3 pos = CameraTransform.position; Quaternion rot = CameraTransform.rotation; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos); } }
void OnHelperPostRender(Ray ray, Vector3 gazepoint) { //RaycastHit hit = new RaycastHit(); Vector3 gpsloc = new Vector3(); float compass = 0; Vector3 floorPos = new Vector3(); GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos); float hitDistance; DynamicObject hitDynamic; Vector3 hitWorld; Vector2 hitcoord; if (DynamicRaycast(ray.origin, ray.direction, CameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitcoord)) //hit dynamic { string ObjectId = hitDynamic.ObjectId.Id; Vector3 LocalGaze = hitDynamic.transform.InverseTransformPointUnscaled(hitWorld); hitDynamic.OnGaze(CognitiveVR_Preferences.S_SnapshotInterval); var mediacomponent = hitDynamic.GetComponent <MediaComponent>(); if (mediacomponent != null) { var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0; var mediauvs = hitcoord; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos); } else { GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, LocalGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, floorPos); } Debug.DrawLine(CameraTransform.position, hitWorld, Color.magenta, 1); return; } if (gazepoint.magnitude > CameraComponent.farClipPlane * 0.99f) //compare to farplane. skybox { Vector3 pos = CameraTransform.position; Quaternion rot = CameraTransform.rotation; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos); Debug.DrawRay(transform.position, transform.forward * CameraComponent.farClipPlane, Color.cyan, 1); } else { Vector3 pos = CameraTransform.position; Quaternion rot = CameraTransform.rotation; //hit world GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos + gazepoint, pos, rot, gpsloc, compass, floorPos); Debug.DrawLine(pos, pos + gazepoint, Color.red, 1); LastGazePoint = pos + gazepoint; } }
//called immediately after construction public virtual void Initialize() { #if CVR_STEAMVR CognitiveVR_Manager.PoseEvent += CognitiveVR_Manager_OnPoseEvent; //1.2 #endif #if CVR_OCULUS OVRManager.HMDMounted += OVRManager_HMDMounted; OVRManager.HMDUnmounted += OVRManager_HMDUnmounted; #endif string hmdname = "none"; #if CVR_FOVE hmdname = "fove"; #elif CVR_ARKIT hmdname = "arkit"; #elif CVR_ARCORE hmdname = "arcore"; #elif CVR_META hmdname = "meta"; #elif UNITY_2019_1_OR_NEWER string rawHMDName = UnityEngine.XR.InputDevices.GetDeviceAtXRNode(UnityEngine.XR.XRNode.Head).name; //string rawHMDName = UnityEngine.XR.XRDevice.model.ToLower(); hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName); #elif UNITY_2017_2_OR_NEWER string rawHMDName = UnityEngine.XR.XRDevice.model.ToLower(); hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName); #else string rawHMDName = UnityEngine.VR.VRDevice.model.ToLower(); hmdname = CognitiveVR.Util.GetSimpleHMDName(rawHMDName); #endif #if CVR_AH ah_calibrator = Calibrator.Instance; #endif #if CVR_PUPIL gazeController = FindObjectOfType <PupilLabs.GazeController>(); if (gazeController != null) { gazeController.OnReceive3dGaze += ReceiveEyeData; } else { Debug.LogError("Pupil Labs GazeController is null!"); } #endif GazeCore.SetHMDType(hmdname); cameraRoot = GameplayReferences.HMD.root; }
private void CognitiveVR_Manager_TickEvent() { if (GameplayReferences.HMD == null) { return; } RaycastHit hit = new RaycastHit(); Ray ray = new Ray(GameplayReferences.HMD.position, GetWorldGazeDirection()); Vector3 gpsloc = new Vector3(); float compass = 0; Vector3 floorPos = new Vector3(); GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos); float hitDistance; DynamicObject hitDynamic; Vector3 hitWorld; Vector3 hitLocal; Vector2 hitcoord; if (DynamicRaycast(ray.origin, ray.direction, GameplayReferences.HMDCameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitLocal, out hitcoord)) //hit dynamic { string ObjectId = hitDynamic.DataId; var mediacomponent = hitDynamic.GetComponent <MediaComponent>(); if (mediacomponent != null) { var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0; var mediauvs = hitcoord; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, GameplayReferences.HMD.position, GameplayReferences.HMD.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos); } else { GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, ray.origin, GameplayReferences.HMD.rotation, gpsloc, compass, floorPos); } Debug.DrawLine(GameplayReferences.HMD.position, hitWorld, new Color(1, 0, 1, 0.5f), CognitiveVR_Preferences.Instance.SnapshotInterval); Debug.DrawRay(hitWorld, Vector3.right, Color.red, 1); Debug.DrawRay(hitWorld, Vector3.forward, Color.blue, 1); Debug.DrawRay(hitWorld, Vector3.up, Color.green, 1); return; } if (Physics.Raycast(ray, out hit, GameplayReferences.HMDCameraComponent.farClipPlane, CognitiveVR_Preferences.Instance.GazeLayerMask)) { Vector3 pos = GameplayReferences.HMD.position; Vector3 gazepoint = hit.point; Quaternion rot = GameplayReferences.HMD.rotation; //hit world GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), gazepoint, pos, rot, gpsloc, compass, floorPos); Debug.DrawLine(pos, gazepoint, Color.red, CognitiveVR_Preferences.Instance.SnapshotInterval); Debug.DrawRay(gazepoint, Vector3.right, Color.red, 10); Debug.DrawRay(gazepoint, Vector3.forward, Color.blue, 10); Debug.DrawRay(gazepoint, Vector3.up, Color.green, 10); } else //hit sky / farclip { Vector3 pos = GameplayReferences.HMD.position; Quaternion rot = GameplayReferences.HMD.rotation; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos); Debug.DrawRay(pos, GameplayReferences.HMD.forward * GameplayReferences.HMDCameraComponent.farClipPlane, Color.cyan, CognitiveVR_Preferences.Instance.SnapshotInterval); } }
private void OnHelperPostRender() { Ray ray = new Ray(CameraTransform.position, GetWorldGazeDirection()); Vector3 gpsloc = new Vector3(); float compass = 0; Vector3 floorPos = new Vector3(); GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos); //raycast for dynamic float hitDistance; DynamicObject hitDynamic; Vector3 hitWorld; string objectId = ""; Vector3 localGaze = Vector3.zero; Vector2 hitcoord; if (DynamicRaycast(ray.origin, ray.direction, CameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitcoord)) //hit dynamic { objectId = hitDynamic.ObjectId.Id; localGaze = hitDynamic.transform.InverseTransformPointUnscaled(hitWorld); } //get depth world point Vector3 point; Vector3 gazedirection = GetWorldGazeDirection(); Vector2 screenGazePoint = GetViewportGazePoint(); bool hitworld = GetGazePoint(Resolution, Resolution, out point, CameraComponent.nearClipPlane, CameraComponent.farClipPlane, CameraTransform.forward, CameraTransform.position, screenGazePoint, gazedirection); float depthDistance = Vector3.Distance(point, CameraTransform.position); if (hitDistance > 0 && hitDistance < depthDistance) //hit a dynamic object closer than the scene depth { //TODO check for media hitDynamic.OnGaze(CognitiveVR_Preferences.S_SnapshotInterval); var mediacomponent = hitDynamic.GetComponent <MediaComponent>(); if (mediacomponent != null) { var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0; var mediauvs = hitcoord; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), objectId, localGaze, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos); } else { GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), objectId, localGaze, ray.origin, CameraTransform.rotation, gpsloc, compass, floorPos); } Debug.DrawLine(CameraTransform.position, hitWorld, Color.magenta, 1); } else //didn't hit a dynamic, or hit one further away than the depth buffer { if (hitworld) //hit world { GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), point, CameraTransform.position, CameraTransform.rotation, gpsloc, compass, floorPos); Debug.DrawLine(CameraTransform.position, point, Color.red, 1); } else //hit skybox { GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), CameraTransform.position, CameraTransform.rotation, gpsloc, compass, floorPos); Debug.DrawRay(CameraTransform.position, CameraTransform.forward * CameraComponent.farClipPlane, Color.cyan, 1); } } }
void OnHelperPostRender(Ray ray, Vector3 gazeVector, Vector3 worldpos) { if (GameplayReferences.HMD == null) { return; } Vector3 gpsloc = new Vector3(); float compass = 0; Vector3 floorPos = new Vector3(); GetOptionalSnapshotData(ref gpsloc, ref compass, ref floorPos); float hitDistance; DynamicObject hitDynamic; Vector3 hitWorld; Vector3 hitLocal; Vector2 hitcoord; string ObjectId = ""; if (DynamicRaycast(ray.origin, ray.direction, GameplayReferences.HMDCameraComponent.farClipPlane, 0.05f, out hitDistance, out hitDynamic, out hitWorld, out hitLocal, out hitcoord)) //hit dynamic { ObjectId = hitDynamic.DataId; } float depthDistance = Vector3.Distance(GameplayReferences.HMD.position, worldpos); if (hitDistance > 0 && hitDistance < depthDistance) { var mediacomponent = hitDynamic.GetComponent <MediaComponent>(); if (mediacomponent != null) { var mediatime = mediacomponent.IsVideo ? (int)((mediacomponent.VideoPlayer.frame / mediacomponent.VideoPlayer.frameRate) * 1000) : 0; var mediauvs = hitcoord; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, GameplayReferences.HMD.position, GameplayReferences.HMD.rotation, gpsloc, compass, mediacomponent.MediaSource, mediatime, mediauvs, floorPos); } else { GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), ObjectId, hitLocal, GameplayReferences.HMD.position, GameplayReferences.HMD.rotation, gpsloc, compass, floorPos); } Debug.DrawLine(GameplayReferences.HMD.position, hitDynamic.transform.position + hitLocal, Color.magenta, 1); Debug.DrawRay(worldpos, Vector3.right, Color.red, 1); Debug.DrawRay(worldpos, Vector3.forward, Color.blue, 1); Debug.DrawRay(worldpos, Vector3.up, Color.green, 1); return; } if (gazeVector.magnitude > GameplayReferences.HMDCameraComponent.farClipPlane * 0.99f) //compare to farplane. skybox { Vector3 pos = GameplayReferences.HMD.position; Quaternion rot = GameplayReferences.HMD.rotation; GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), pos, rot, gpsloc, compass, floorPos); Debug.DrawRay(pos, GameplayReferences.HMD.forward * GameplayReferences.HMDCameraComponent.farClipPlane, Color.cyan, 1); } else { Vector3 pos = GameplayReferences.HMD.position; Quaternion rot = GameplayReferences.HMD.rotation; //hit world GazeCore.RecordGazePoint(Util.Timestamp(Time.frameCount), worldpos, pos, rot, gpsloc, compass, floorPos); //Debug.DrawLine(pos, pos + gazepoint, Color.red, 1); Debug.DrawLine(ray.origin, worldpos, Color.yellow, 1); Debug.DrawRay(worldpos, Vector3.right, Color.red, 1); Debug.DrawRay(worldpos, Vector3.forward, Color.blue, 1); Debug.DrawRay(worldpos, Vector3.up, Color.green, 1); //Debug.Log("hit world"); LastGazePoint = worldpos; } }