// Latepdate ensures that the object doesn't lag behind the user's head motion
void Update()
{
FoveInterfaceBase.EyeRays rays = foveInterface.GetGazeRays();
Ray r = whichEye == LeftOrRight.Left ? rays.left : rays.right;
RaycastHit hit;
Physics.Raycast(r, out hit, Mathf.Infinity);
if (hit.point != Vector3.zero) // Vector3 is non-nullable; comparing to null is always false
{
transform.position = hit.point;
if (frame % 2 == 0)
{
if (hit.collider.tag != "cubes")
{
GameObject cube = GameObject.CreatePrimitive(PrimitiveType.Cube);
cube.transform.localScale = new Vector3(0.2f, 0.2f, 0.01f);
cube.transform.position = hit.point;
cube.GetComponent <Renderer>().material.color = Color.white;
cube.GetComponent <Collider>().tag = "cubes";
}
else
{
Color c = hit.collider.GetComponent <Renderer>().material.color;
hit.collider.GetComponent <Renderer>().material.color = new Color(c.r - 0.08f, c.g - 0.08f, c.b - 0.08f);
}
}
}
else
{
transform.position = r.GetPoint(3.0f);
}
frame++;
}
//Returns the position on the texuremap according to a hit in the mesh collider
bool HitTestUVPosition(ref Vector3 uvWorldPosition)
{
RaycastHit hit;
Vector3 cursorPos = new Vector3(Input.mousePosition.x, Input.mousePosition.y, 0.0f);
FoveInterfaceBase.EyeRays rays = foveInterface.GetGazeRays();
//Ray cursorRay=sceneCamera.ScreenPointToRay (cursorPos);
Ray cursorRay = whichEye == LeftOrRight.Left ? rays.left : rays.right;
if (Physics.Raycast(cursorRay, out hit, 200))
{
MeshCollider meshCollider = hit.collider as MeshCollider;
if (meshCollider == null || meshCollider.sharedMesh == null)
{
return(false);
}
Vector2 pixelUV = new Vector2(hit.textureCoord.x, hit.textureCoord.y);
uvWorldPosition.x = pixelUV.x - canvasCam.orthographicSize; //To center the UV on X
uvWorldPosition.y = pixelUV.y - canvasCam.orthographicSize; //To center the UV on Y
uvWorldPosition.z = 0.0f;
Debug.Log("pos " + pixelUV.x + "," + pixelUV.y);
return(true);
}
else
{
return(false);
}
}
// Latepdate ensures that the object doesn't lag behind the user's head motion
void Update()
{
FoveInterfaceBase.EyeRays rays = foveInterface.GetGazeRays();
Ray r = whichEye == LeftOrRight.Left ? rays.left : rays.right;
RaycastHit hit;
Physics.Raycast(r, out hit, Mathf.Infinity);
if (hit.point != Vector3.zero) // Vector3 is non-nullable; comparing to null is always false
{
transform.position = hit.point;
}
else
{
transform.position = r.GetPoint(5.0f);
}
}
private void gazeOnWall()
{
FoveInterfaceBase.EyeRays rays = foveInterface.GetGazeRays();
Ray r = whichEye == Direction.Left ? rays.left : rays.right;
RaycastHit hit;
Physics.Raycast(r, out hit, Mathf.Infinity);
int rate = conf.getConfiguration().rate;
if ((hit.point != Vector3.zero) && (countForLocations.Elapsed.TotalSeconds > rate))
{
locationList.Add(new location(hit.point.x, hit.point.y, CommonData.realStep));
countForLocations.Stop();
countForLocations.Reset();
countForLocations.Start();
}
}
// Postprocess the image
void OnRenderImage(RenderTexture source, RenderTexture destination)
{
// If the shaders are deactivated, just show the normal scene
if (shaders)
{
if (blacken)
{
mat_main.SetInt("_is_black", 1);
Graphics.Blit(source, destination, mat_main);
}
else
{
// FOVE eye tracking calculations
FoveInterfaceBase.EyeRays rays = foveInterface.GetGazeRays();
Ray r = whichEye == LeftOrRight.Left ? rays.left : rays.right;
if (trackingMode == TrackMode.ViewArea)
{
RaycastHit hit;
Physics.Raycast(r, out hit, Mathf.Infinity);
Vector2 center = new Vector2(((float)(cam.WorldToScreenPoint(hit.point).x) / (float)cam.pixelWidth),
((float)(cam.WorldToScreenPoint(hit.point).y) / (float)cam.pixelHeight));
mat_main.SetFloat("_centerx", center.x);
mat_main.SetFloat("_centery", center.y);
}
else
{
cam.transform.rotation = Quaternion.LookRotation(r.direction);
}
// Calculations done by hand for 60fps. It should be 10ms of fire and 40 of silence (1 frame every 5 at 100Hz)
// right now it's 16.6/50
fCount = (fCount + 1) % pulseCycle;
if (fCount == 0)
{
mat_main.SetInt("_is_black", 0);
}
else
{
mat_main.SetInt("_is_black", 1);
}
if (dynamicDisturbance)
{
mat_main.SetFloat("_seed_mask", Random.value);
}
// I am not 100% sure why, but it works well, without distorting. I assume that the display is virtually split in two square-ish halves
if (canny)
{
// Fove's resolution
RenderTexture tmp1 = RenderTexture.GetTemporary(2560, 1440);
Graphics.Blit(source, tmp1, mat_canny);
Graphics.Blit(tmp1, destination, mat_main);
RenderTexture.ReleaseTemporary(tmp1);
}
else
{
Graphics.Blit(source, destination, mat_main);
}
}
}
else
{
if (canny)
{
Graphics.Blit(source, destination, mat_canny);
}
else
{
Graphics.Blit(source, destination);
}
}
}
//Gets point where user is looking every frame and interacts with any intersecting gazeobjects if possible
void Update()
{
if (!_initialized)
{
return;
}
if (Input.GetKeyDown(KeyCode.H))
{
CenterHead();
}
if (Input.GetKeyDown(KeyCode.Escape))
{
Application.Quit();
}
Ray ray = new Ray();
switch (_selectedControlType)
{
case StreamController.ControlType.Head:
ray = new Ray(Head.position, Head.forward * 1000);
break;
case StreamController.ControlType.Eyes_Mouse:
case StreamController.ControlType.Mouse:
if (Input.GetMouseButtonDown(1))
{
}
if (Input.GetMouseButton(1))
{
Head.Rotate(Vector3.up, Input.GetAxis("Mouse X") * _mouseRotationSpeed, Space.Self);
Head.Rotate(Vector3.right, -Input.GetAxis("Mouse Y") * _mouseRotationSpeed, Space.Self);
Head.localRotation = Quaternion.Euler(Head.localEulerAngles.x, Head.localEulerAngles.y, 0);
}
if (Input.GetMouseButton(0) || _selectedControlType == StreamController.ControlType.Eyes_Mouse)
{
ray = Camera.main.ScreenPointToRay(Input.mousePosition);
}
else
{
ResetHoveredObject();
return;
}
break;
//both of the code for the two input cases was moved further down, since we want gaze data to be recorded for both inputs.
case StreamController.ControlType.Eyes:
//List<Vector3> eyeDirections = new List<Vector3>();
//FoveInterfaceBase.EyeRays rays = _foveInterface.GetGazeRays();
//EFVR_Eye eyeClosed = FoveInterface.CheckEyesClosed();
//if (eyeClosed != EFVR_Eye.Both && eyeClosed != EFVR_Eye.Left)
// eyeDirections.Add(rays.left.direction);
//if (eyeClosed != EFVR_Eye.Both && eyeClosed != EFVR_Eye.Right)
// eyeDirections.Add(rays.right.direction);
//Vector3 direction = Vector3.zero;
//foreach (Vector3 eyeDirection in eyeDirections)
//{
// direction += eyeDirection;
//}
//direction = direction / eyeDirections.Count;
//ray = new Ray(Head.transform.position, direction * 1000);
break;
case StreamController.ControlType.Joystick:
{
// // Joystick input
//Vector2 JoyInput = new Vector2(Input.GetAxis("Horizontal"), Input.GetAxis("Vertical"));
////if the virtual environment is on, send the command to the VirtualUnityController
//if (StreamController.Instance.VirtualEnvironment)
//{
// if (VirtualUnityController.Instance.IsActive)
// {
// VirtualUnityController.Instance.JoystickCommand(JoyInput);
// }
//}
//// Othewise send it to the robotinterface
//else
//{
// if (RobotInterface.Instance.IsConnected)
// {
// RobotInterface.Instance.DirectCommandRobot(JoyInput);
// }
//}
break;
}
}
//--Eye direction calculation for all occasions
List <Vector3> eyeDirections = new List <Vector3>();
FoveInterfaceBase.EyeRays rays = _foveInterface.GetGazeRays();
EFVR_Eye eyeClosed = FoveInterface.CheckEyesClosed();
if (eyeClosed != EFVR_Eye.Both && eyeClosed != EFVR_Eye.Left)
{
eyeDirections.Add(rays.left.direction);
}
if (eyeClosed != EFVR_Eye.Both && eyeClosed != EFVR_Eye.Right)
{
eyeDirections.Add(rays.right.direction);
}
Vector3 direction = Vector3.zero;
foreach (Vector3 eyeDirection in eyeDirections)
{
direction += eyeDirection;
}
direction = direction / eyeDirections.Count;
ray = new Ray(Head.transform.position, direction * 1000);
//---------------------------------------------------------
//Positioning of the cursor
_cursorCanvas.position = Head.position + ray.direction * _cursorDistance;
Debug.DrawRay(ray.origin, ray.direction);
RaycastHit hit;
if (Physics.Raycast(ray, out hit))
{
GazeObject gazeObject = hit.collider.GetComponent <GazeObject>();
if (gazeObject == null)
{
ResetHoveredObject();
return;
}
// For this reason we also check if the tag of the gazeobject is the correct one
RobotControlTrackPad robotControl = gazeObject.GetComponent <RobotControlTrackPad>();
if (robotControl != null && gazeObject.CompareTag("EyeControlPanel"))
{
//Control result is provided on hit. This is updated for both cases of input
controlResult = robotControl.GetControlResult(hit.point);
//If the robotcontrols are activated and the eye tracking is used for motion then send the command to the appropriate controller
if (robotControl.IsActivated & !robotControl.IsExternallyDisabled() &&
_selectedControlType == StreamController.ControlType.Eyes)
{
if (StreamController.Instance.VirtualEnvironment)
{
if (VirtualUnityController.Instance.IsActive)
{
// Debug.Log("Sending gaze command to robot");
VirtualUnityController.Instance.GazeCommand(controlResult);
}
else
{
Debug.Log("VirtualUnityController is not connected");
}
}
// Othewise send it to the robotinterface
else
{
if (RobotInterface.Instance.IsConnected)
{
RobotInterface.Instance.SendCommand(controlResult);
}
else
{
Debug.Log("RobotInterface controller is not connected");
}
}
//Instead of robotinterface here
}
//---Joystick Input---
else if (robotControl.IsActivated & !robotControl.IsExternallyDisabled() &&
_selectedControlType == StreamController.ControlType.Joystick)
{
// Joystick input
Vector2 JoyInput = new Vector2(Input.GetAxis("Horizontal"), Input.GetAxis("Vertical"));
//if the virtual environment is on, send the command to the VirtualUnityController
if (StreamController.Instance.VirtualEnvironment)
{
if (VirtualUnityController.Instance.IsActive)
{
VirtualUnityController.Instance.JoystickCommand(JoyInput);
}
}
// Othewise send it to the robotinterface
else
{
if (RobotInterface.Instance.IsConnected)
{
RobotInterface.Instance.DirectCommandRobot(JoyInput);
}
}
}
}
else
{
//this result means not staring at panel.
controlResult = new Vector2(-2, -2);
//TODO : SendStopCommandToRobot instead of a zero vector. The zero vector is filtered and still adds movemenet to the robot
// RobotInterface.Instance.SendCommand(Vector2.zero);
}
if (gazeObject == _hoveredGazeObject)
{
return;
}
if (_hoveredGazeObject != null)
{
_hoveredGazeObject.OnUnhover();
}
gazeObject.OnHover();
_hoveredGazeObject = gazeObject;
}
else
{
ResetHoveredObject();
}
}
// Update is called once per frame
void Update()
{
Vector3 pos = transform.position;
SFVR_Quaternion orientation = FoveInterface.GetLastPose().orientation;
Quaternion quat = new Quaternion(orientation.x, orientation.y, orientation.z, orientation.w);
Vector3 forwad = quat * Vector3.forward;
if (_shouldStop)
{
if (_shouldLeftEyeDark)
{
foveInterface.LeftFoveEye.ShouldDraw = false;
foveInterface.RightFoveEye.ShouldDraw = true;
}
else
{
foveInterface.LeftFoveEye.ShouldDraw = true;
foveInterface.RightFoveEye.ShouldDraw = false;
}
_shouldEyeDarkTimer += Time.deltaTime;
if (_shouldEyeDarkTimer > 3.0f)
{
_shouldEyeDarkTimer = 0.0f;
_shouldLeftEyeDark = !_shouldLeftEyeDark;
if (_shouldLeftEyeDark)
{
print("left flick");
}
else
{
print("right flick");
}
}
}
else
{
float speed = 20;
distanceThreshold += speed * Time.deltaTime;
if (goForward && distanceThreshold > speed * 5)
{
distanceThreshold = 0.0f;
goForward = false;
_shouldStop = true;
}
else if (!goForward && distanceThreshold > speed * 5)
{
distanceThreshold = 0.0f;
goForward = true;
}
if (goForward)
{
pos += forwad * speed * Time.deltaTime;
}
else
{
pos -= forwad * speed * Time.deltaTime;
}
transform.position = pos;
}
FoveInterfaceBase.EyeRays eyeRays = foveInterface.GetGazeRays();
RaycastHit leftRaycastHit, rightRaycastHit;
Physics.Raycast(eyeRays.left, out leftRaycastHit, Mathf.Infinity);
if (leftRaycastHit.point != Vector3.zero)
{
leftEye.transform.position = leftRaycastHit.point;
}
else
{
leftEye.transform.position = eyeRays.left.GetPoint(3.0f);
}
_leftRecords.Add(new EyeTrackingRecord(Timer, eyeRays.left.origin, eyeRays.left.direction, leftEye.transform.position));
Physics.Raycast(eyeRays.right, out rightRaycastHit, Mathf.Infinity);
if (rightRaycastHit.point != Vector3.zero)
{
rightEye.transform.position = rightRaycastHit.point;
}
else
{
rightEye.transform.position = eyeRays.right.GetPoint(3.0f);
}
_rightRecords.Add(new EyeTrackingRecord(Timer, eyeRays.right.origin, eyeRays.right.direction, rightEye.transform.position));
if (SaveTimer > 1.0f)
{
string leftPath = LoggingManager.GetPath(dataFileID + "_left.csv");
System.IO.FileInfo file = new System.IO.FileInfo(leftPath);
file.Directory.Create();
using (var writer = new StreamWriter(leftPath, append: true))
{
foreach (EyeTrackingRecord record in _leftRecords)
{
writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
record.Timer,
record.Origin.x,
record.Origin.y,
record.Origin.z,
record.Direction.x,
record.Direction.y,
record.Direction.z,
record.TouchedPosition.x,
record.TouchedPosition.y,
record.TouchedPosition.z
);
}
writer.Flush();
_leftRecords.Clear();
}
string rightPath = LoggingManager.GetPath(dataFileID + "_right.csv");
file = new System.IO.FileInfo(rightPath);
file.Directory.Create();
using (var writer = new StreamWriter(rightPath, append: true))
{
foreach (EyeTrackingRecord record in _rightRecords)
{
writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
record.Timer,
record.Origin.x,
record.Origin.y,
record.Origin.z,
record.Direction.x,
record.Direction.y,
record.Direction.z,
record.TouchedPosition.x,
record.TouchedPosition.y,
record.TouchedPosition.z
);
}
writer.Flush();
_rightRecords.Clear();
}
SaveTimer = 0.0f;
}
SaveTimer += Time.deltaTime;
Timer += Time.deltaTime;
}
// Update is called once per frame
void Update()
{
FoveInterfaceBase.EyeRays eyeRays = foveInterface.GetGazeRays();
RaycastHit leftRaycastHit, rightRaycastHit;
Physics.Raycast(eyeRays.left, out leftRaycastHit, Mathf.Infinity);
if (leftRaycastHit.point != Vector3.zero)
{
leftEye = leftRaycastHit.point;
}
else
{
leftEye = eyeRays.left.GetPoint(3.0f);
}
_leftRecords.Add(new EyeTrackingRecord(Timer, eyeRays.left.origin, eyeRays.left.direction, leftEye));
Physics.Raycast(eyeRays.right, out rightRaycastHit, Mathf.Infinity);
if (rightRaycastHit.point != Vector3.zero)
{
rightEye = rightRaycastHit.point;
}
else
{
rightEye = eyeRays.right.GetPoint(3.0f);
}
_rightRecords.Add(new EyeTrackingRecord(Timer, eyeRays.right.origin, eyeRays.right.direction, rightEye));
if (SaveTimer > 1.0f)
{
string leftPath = LoggingManager.GetPath(dataFileID + "_left.csv");
System.IO.FileInfo file = new System.IO.FileInfo(leftPath);
file.Directory.Create();
using (var writer = new StreamWriter(leftPath, append: true))
{
foreach (EyeTrackingRecord record in _leftRecords)
{
writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
record.Timer,
record.Origin.x,
record.Origin.y,
record.Origin.z,
record.Direction.x,
record.Direction.y,
record.Direction.z,
record.TouchedPosition.x,
record.TouchedPosition.y,
record.TouchedPosition.z
);
}
writer.Flush();
_leftRecords.Clear();
}
string rightPath = LoggingManager.GetPath(dataFileID + "_right.csv");
file = new System.IO.FileInfo(rightPath);
file.Directory.Create();
using (var writer = new StreamWriter(rightPath, append: true))
{
foreach (EyeTrackingRecord record in _rightRecords)
{
writer.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
record.Timer,
record.Origin.x,
record.Origin.y,
record.Origin.z,
record.Direction.x,
record.Direction.y,
record.Direction.z,
record.TouchedPosition.x,
record.TouchedPosition.y,
record.TouchedPosition.z
);
}
writer.Flush();
_rightRecords.Clear();
}
SaveTimer = 0.0f;
}
SaveTimer += Time.deltaTime;
Timer += Time.deltaTime;
}
private void Update()
{
// update global eye positions
FoveInterfaceBase.EyeRays rays = foveInterface.GetGazeRays();
left_eye_vector = rays.left.direction;
right_eye_vector = rays.right.direction;
// always Serial send to Arduino
double theta_L = (180.0 / 3.14159265) * -Math.Atan2((double)left_eye_vector.x, (double)left_eye_vector.z);
double rho_L = (180.0 / 3.14159265) * Math.Atan2((double)left_eye_vector.y, (double)Math.Sqrt(left_eye_vector.x * left_eye_vector.x + left_eye_vector.z * left_eye_vector.z));
// Debug.Log(theta_L);
// Debug.Log(rho_L);
// converts stuff into bytes
string dataString = theta_L.ToString() + "," + rho_L.ToString() + ";";
// Debug.Log(dataString);
arduinoPort.Write(dataString);
/*
* // Debug.Log(dataString);
* byte[] send_buffer = Encoding.ASCII.GetBytes(dataString);
* //byte[] unicodeBytes = Encoding.Unicode.GetBytes(unicodeString);
* //byte[] asciiBytes = Encoding.Convert(Encoding.Unicode, Encoding.ASCII, unicodeBytes);
* udp_kofiko.Send(send_buffer, send_buffer.Length);
* // Debug.Log("done sending!");
*/
if (triggered)
{
if (show_surface)
{
// float orientation_y = rotation_amount * (2 * UnityEngine.Random.value - 1);
// float orientation_z = 90 + rotation_amount * (2 * UnityEngine.Random.value - 1);
float orientation_x = -90 + rotation_amount * (2 * rotation_x - 1);
float orientation_y = rotation_amount * (2 * rotation_y - 1);
Quaternion rotation = Quaternion.identity;
rotation.eulerAngles = new Vector3(orientation_x, orientation_y, 0);
GameObject.Find("surface").transform.localRotation = rotation;
// change position
// float displacement_y = displacement_magnitude * (2 * UnityEngine.Random.value - 1);
// float displacement_z = displacement_magnitude * (2 * UnityEngine.Random.value - 1);
float displacement_y = displacement_magnitude * (2 * translation_y - 1);
float displacement_x = displacement_magnitude * (2 * translation_x - 1);
GameObject.Find("surface").transform.localPosition = new Vector3(displacement_x, displacement_y, z_distance);
// make appear
// GameObject.Find ("fixation cross").transform.localScale = init_fixation_point_size*disappear_scale;
GameObject.Find("surface").transform.localScale = init_surface_size;
triggered = false;
}
else
{
GameObject.Find("surface").transform.localScale = init_surface_size * disappear_scale;
}
if (show_calibration)
{
// from angles: 1040, 540 to points ...
float theta = 3.14159265f * ((translation_x - 540) / 540);
float rho = 3.14159265f * (translation_y / 540);;
float displacement_z = (float)(calibration_disparity * Math.Sin(rho) * Math.Cos(theta));
float displacement_x = (float)(calibration_disparity * Math.Sin(rho) * Math.Sin(theta));
float displacement_y = (float)(calibration_disparity * Math.Cos(rho));
// calculate new coordinates
GameObject.Find("calibration point").transform.localPosition = new Vector3(displacement_x, displacement_y, displacement_z);
GameObject.Find("calibration point").transform.localScale = z_distance * 3.14159265f * (calibration_scale / 540) * (calibration_disparity / 7) * new Vector3(1.0f, 1.0f, 1.0f);
}
else
{
GameObject.Find("calibration point").transform.localScale = init_surface_size * disappear_scale;
}
if (show_cross)
{
GameObject.Find("fixation cross").transform.localScale = init_cross_size;
}
else
{
GameObject.Find("fixation cross").transform.localScale = init_cross_size * disappear_scale;
}
}
/*
* if (Input.GetKeyDown("space")) {
* print("space key was pressed") ;
*
* if (show_surface) {
* // make disappear
* GameObject.Find ("surface").transform.localScale *= disappear_scale;
* // GameObject.Find ("fixation cross").transform.localScale /= disappear_scale;
* } else {
* // change orientation
* float orientation_y = rotation_amount * (2 * UnityEngine.Random.value - 1);
* float orientation_z = 90 + rotation_amount * (2 * UnityEngine.Random.value - 1);
* Quaternion rotation = Quaternion.identity;
* rotation.eulerAngles = new Vector3(0, orientation_y, orientation_z) ;
* GameObject.Find ("surface").transform.localRotation = rotation ;
*
* // change position
* float displacement_y = displacement_magnitude * (2 * UnityEngine.Random.value - 1);
* float displacement_z = displacement_magnitude * (2 * UnityEngine.Random.value - 1);
* GameObject.Find ("surface").transform.localPosition = new Vector3(x_distance, displacement_y, displacement_z);
*
* // make appear
* // GameObject.Find ("fixation cross").transform.localScale *= disappear_scale;
* GameObject.Find ("surface").transform.localScale /= disappear_scale;
* }
* show_surface = !(show_surface);
* }
*
*/
return;
}
// Update is called once per frame
void Update()
{
//ray = Camera.main.ScreenPointToRay(Input.mousePosition);
rays = foveInterface.GetGazeRays();
}
