void NetMQClient()
{
//thanks for Yuta Itoh sample code to connect via NetMQ with Pupil Service
string IPHeader = ">tcp://" + ServerIP + ":";
var timeout = new System.TimeSpan(0, 0, 1); //1sec
// Necessary to handle this NetMQ issue on Unity editor
// https://github.com/zeromq/netmq/issues/526
AsyncIO.ForceDotNet.Force();
NetMQConfig.ManualTerminationTakeOver();
NetMQConfig.ContextCreate(true);
string subport = "";
Debug.Log("Connect to the server: " + IPHeader + ServicePort + ".");
_requestSocket = new RequestSocket(IPHeader + ServicePort);
_requestSocket.SendFrame("SUB_PORT");
_isconnected = _requestSocket.TryReceiveFrameString(timeout, out subport);
_lastT = DateTime.Now;
if (_isconnected)
{
StartProcess();
var subscriberSocket = new SubscriberSocket(IPHeader + subport);
subscriberSocket.Subscribe("gaze"); //subscribe for gaze data
subscriberSocket.Subscribe("notify."); //subscribe for all notifications
_setStatus(EStatus.ProcessingGaze);
var msg = new NetMQMessage();
while (_isDone == false)
{
_isconnected = subscriberSocket.TryReceiveMultipartMessage(timeout, ref (msg));
if (_isconnected)
{
try
{
string msgType = msg[0].ConvertToString();
//Debug.Log(msgType);
if (msgType == "gaze")
{
var message = MsgPack.Unpacking.UnpackObject(msg[1].ToByteArray());
MsgPack.MessagePackObject mmap = message.Value;
lock (_dataLock)
{
_pupilData = JsonUtility.FromJson <Pupil.PupilData3D>(mmap.ToString());
if (_pupilData.confidence > 0.5f)
{
OnPacket(_pupilData);
}
}
}
//Debug.Log(message);
}
catch
{
// Debug.Log("Failed to unpack.");
}
}
else
{
// Debug.Log("Failed to receive a message.");
Thread.Sleep(500);
}
}
StopProcess();
subscriberSocket.Close();
}
else
{
Debug.Log("Failed to connect the server.");
}
_requestSocket.Close();
// Necessary to handle this NetMQ issue on Unity editor
// https://github.com/zeromq/netmq/issues/526
Debug.Log("ContextTerminate.");
NetMQConfig.ContextTerminate();
}
void OnPacket(Pupil.PupilData3D data)
{
//add new frame
_gazeFPS++;
var ct = DateTime.Now;
if ((ct - _lastT).TotalSeconds > 1)
{
_lastT = ct;
_currentFps = _gazeFPS;
_gazeFPS = 0;
}
if (m_status == EStatus.ProcessingGaze) //gaze processing stage
{
float x, y;
x = (float)data.norm_pos [0];
y = (float)data.norm_pos [1];
_eyePos.x = (leftEye.gaze.x + rightEye.gaze.x) * 0.5f;
_eyePos.y = (leftEye.gaze.y + rightEye.gaze.y) * 0.5f;
if (data.id == 0)
{
leftEye.AddGaze(x, y);
if (OnEyeGaze != null)
{
OnEyeGaze(this);
}
}
else if (data.id == 1)
{
rightEye.AddGaze(x, y);
if (OnEyeGaze != null)
{
OnEyeGaze(this);
}
}
}
else if (m_status == EStatus.Calibration) //gaze calibration stage
{
float t = GetPupilTimestamp();
var ref0 = new Dictionary <string, object>()
{
{ "norm_pos", new float[] { _calibPoints[_currCalibPoint].x, _calibPoints[_currCalibPoint].y } }, { "timestamp", t }, { "id", 0 }
};
var ref1 = new Dictionary <string, object>()
{
{ "norm_pos", new float[] { _calibPoints[_currCalibPoint].x, _calibPoints[_currCalibPoint].y } }, { "timestamp", t }, { "id", 1 }
};
_CalibData(_calibPoints [_currCalibPoint].x, _calibPoints [_currCalibPoint].y);
_calibrationData.Add(ref0);
_calibrationData.Add(ref1);
_currCalibSamples++;
Thread.Sleep(1000 / 60);
if (_currCalibSamples >= _calibSamples)
{
_currCalibSamples = 0;
_currCalibPoint++;
string pointsData = "[";
int index = 0;
foreach (var v in _calibrationData)
{
pointsData += JsonUtility.ToJson(v); //String.Format("{'norm_pos':({0},{1}),'timestamp':{2},'id':{3}}",v.norm_pos[0],v.norm_pos[1],v.timestamp,v.id);
++index;
if (index != _calibrationData.Count)
{
pointsData += ",";
}
}
pointsData += "]";
// pointsData = JsonUtility.ToJson (_CalibrationPoints);
//Debug.Log (pointsData);
_sendRequestMessage(new Dictionary <string, object> {
{ "subject", "calibration.add_ref_data" }, { "ref_data", _CalibrationPoints }
});
_calibrationData.Clear();
if (_currCalibPoint >= _calibPoints.Length)
{
StopCalibration();
}
}
}
}
void OnPacket(Pupil.PupilData3D data)
{
//add new frame
_gazeFps.AddFrame();
if (m_status == EStatus.ProcessingGaze) //gaze processing stage
{
int id = data.base_data [0].id;
float x, y;
x = (float)data.norm_pos [0];
y = (float)data.norm_pos [1];
_eyePos.x = (leftEye.gaze.x + rightEye.gaze.x) * 0.5f;
_eyePos.y = (leftEye.gaze.y + rightEye.gaze.y) * 0.5f;
if (id == 0)
{
_eyeFps[0].AddFrame();
leftEye.AddGaze(x, y);
if (OnEyeGaze != null)
{
OnEyeGaze(this, id);
}
}
else if (id == 1)
{
_eyeFps[1].AddFrame();
rightEye.AddGaze(x, y);
if (OnEyeGaze != null)
{
OnEyeGaze(this, id);
}
}
}
else if (m_status == EStatus.Calibration) //gaze calibration stage
{
float t = GetPupilTimestamp();
floatArray[] _cPoints = GetCalibPoints;
float[] _cPointFloatValues = _cPoints [_currCalibPoint].axisValues;
var ref0 = new Dictionary <string, object>()
{
{ "norm_pos", _cPointFloatValues }, { "timestamp", t }, { "id", 0 }
};
var ref1 = new Dictionary <string, object>()
{
{ "norm_pos", _cPointFloatValues }, { "timestamp", t }, { "id", 1 }
};
//keeping this until the new calibration method is not yet tested
// var ref0=new Dictionary<string,object>(){{"norm_pos",new float[]{_calibPoints[_currCalibPoint].x,_calibPoints[_currCalibPoint].y}},{"timestamp",t},{"id",0}};
// var ref1=new Dictionary<string,object>(){{"norm_pos",new float[]{_calibPoints[_currCalibPoint].x,_calibPoints[_currCalibPoint].y}},{"timestamp",t},{"id",1}};
//If OnCalibData delegate has assigned function from the Calibration Marker, assign the current calibration position to it.
_CalibData(_cPointFloatValues);
_calibrationData.Add(ref0);
_calibrationData.Add(ref1);
//Increment the current calibration sample. (Default sample amount per calibration point is 120)
_currCalibSamples++;
print("Sampling at : " + _currCalibSamples);
//give a small timeout per sample.
Thread.Sleep(1000 / 60);
//If the current calibration sample is bigger or equal to the desired sampling (so we accomplished sampling for this calibration point),
//null the current sample and step to next calbration point.
//Also prepare calibration data for sending, and send it.
if (_currCalibSamples >= _calibSamples)
{
_currCalibSamples = 0;
_currCalibPoint++;
//reformat the calibration data for sending.
string pointsData = "[";
int index = 0;
foreach (var v in _calibrationData)
{
pointsData += JsonUtility.ToJson(v); //String.Format("{'norm_pos':({0},{1}),'timestamp':{2},'id':{3}}",v.norm_pos[0],v.norm_pos[1],v.timestamp,v.id);
++index;
if (index != _calibrationData.Count)
{
pointsData += ",";
}
}
pointsData += "]";
// pointsData = JsonUtility.ToJson (_CalibrationPoints);
//Debug.Log (pointsData);
//Send the current relevant calibration data for the current calibration point.
_sendRequestMessage(new Dictionary <string, object> {
{ "subject", "calibration.add_ref_data" }, { "ref_data", _CalibrationPoints }
});
//Clear the current calibration data, so we can proceed to the next point if there is any.
_calibrationData.Clear();
//Stop calibration if we accomplished all required calibration target.
if (_currCalibPoint >= _cPoints.Length)
{
StopCalibration();
}
}
}
}
void NetMQClient()
{
//thanks for Yuta Itoh sample code to connect via NetMQ with Pupil Service
string IPHeader = ">tcp://" + ServerIP + ":";
var timeout = new System.TimeSpan(0, 0, 1); //1sec
// Necessary to handle this NetMQ issue on Unity editor
// https://github.com/zeromq/netmq/issues/526
AsyncIO.ForceDotNet.Force();
NetMQConfig.ManualTerminationTakeOver();
NetMQConfig.ContextCreate(true);
string subport = "";
print("Connect to the server: " + IPHeader + ServicePort + ".");
Thread.Sleep(ServiceStartupDelay);
_requestSocket = new RequestSocket(IPHeader + ServicePort);
_requestSocket.SendFrame("SUB_PORT");
_isconnected = _requestSocket.TryReceiveFrameString(timeout, out subport);
print(_isconnected + " isconnected");
_gazeFps.Reset();
_eyeFps [0].Reset();
_eyeFps [1].Reset();
if (_isconnected)
{
//_serviceStarted = true;
StartProcess();
var subscriberSocket = new SubscriberSocket(IPHeader + subport);
subscriberSocket.Subscribe("gaze"); //subscribe for gaze data
subscriberSocket.Subscribe("notify."); //subscribe for all notifications
_setStatus(EStatus.ProcessingGaze);
var msg = new NetMQMessage();
while (_isDone == false)
{
_isconnected = subscriberSocket.TryReceiveMultipartMessage(timeout, ref (msg));
if (_isconnected)
{
try
{
string msgType = msg[0].ConvertToString();
UnityEngine.Debug.Log(msgType);
if (msgType == "gaze")
{
var message = MsgPack.Unpacking.UnpackObject(msg[1].ToByteArray());
MsgPack.MessagePackObject mmap = message.Value;
lock (_dataLock)
{
_pupilData = JsonUtility.FromJson <Pupil.PupilData3D>(mmap.ToString());
if (_pupilData.confidence > 0.5f)
{
//UnityEngine.Debug.Log(_pupilData.base_data[0].id);
OnPacket(_pupilData);
}
}
}
else if (msgType == "notify.eye_process.started")
{
var message = MsgPack.Unpacking.UnpackObject(msg[1].ToByteArray());
MsgPack.MessagePackObject mmap = message.Value;
var id = JsonUtility.FromJson <Pupil.EyeStatus>(mmap.ToString());
UnityEngine.Debug.Log(id.eye_id);
}
//Debug.Log(message);
}
catch
{
// Debug.Log("Failed to unpack.");
}
}
else
{
print("Failed to receive a message.");
Thread.Sleep(500);
}
}
StopProcess();
subscriberSocket.Close();
}
else
{
print("Failed to connect the server.");
//If needed here could come a retry connection.
}
//Can only send request via IPC if the connection has been established, otherwise we are facing, errors and potential freezing.
if (_serviceStarted && _isconnected)
{
StopService();
}
//Kill process
if (serviceProcess != null)
{
UnityEngine.Debug.Log("Killing Pupil service");
serviceProcess.Kill();
serviceProcess.Close();
}
_requestSocket.Close();
// Necessary to handle this NetMQ issue on Unity editor
// https://github.com/zeromq/netmq/issues/526
print("ContextTerminate.");
NetMQConfig.ContextTerminate();
}
void OnPacket(Pupil.PupilData3D data)
{
float x, y;
_gazeFPS++;
ct = DateTime.Now;
if ((ct - _lastT).TotalSeconds > 1)
{
_lastT = ct;
_currentFps = _gazeFPS;
_gazeFPS = 0;
}
if (m_status == EStatus.ProcessingGaze) //gaze processing stage
{
x = (float)data.norm_pos [0];
y = (float)data.norm_pos [1];
if (data.id == 0)
{
leftEye.AddGaze(x, y);
}
else if (data.id == 1)
{
rightEye.AddGaze(x, y);
}
centerEye.x = (leftEye.gaze.x + rightEye.gaze.x) * 0.5f;
centerEye.y = (leftEye.gaze.y + rightEye.gaze.y) * 0.5f;
}
else if (m_status == EStatus.Calibration) //gaze calibration stage
{
// Get Pupil Timestamp
_requestSocket.SendFrame("t");
NetMQMessage recievedMsg = _requestSocket.ReceiveMultipartMessage();
float t = float.Parse(recievedMsg[0].ConvertToString());
var ref0 = new Dictionary <string, object>()
{
{ "norm_pos", new float[] { _calibPoints[_currCalibPoint].x, _calibPoints[_currCalibPoint].y } }, { "timestamp", t }, { "id", 0 }
};
var ref1 = new Dictionary <string, object>()
{
{ "norm_pos", new float[] { _calibPoints[_currCalibPoint].x, _calibPoints[_currCalibPoint].y } }, { "timestamp", t }, { "id", 1 }
};
if (OnCalibrationData != null)
{
OnCalibrationData(this, _calibPoints [_currCalibPoint].x, _calibPoints [_currCalibPoint].y);
}
_calibrationData.Add(ref0);
_calibrationData.Add(ref1);
_currCalibSamples++;
Thread.Sleep(1000 / 60);
if (_currCalibSamples >= DefaultCalibrationCount)
{
_currCalibSamples = 0;
_currCalibPoint++;
_sendRequestMessage(new Dictionary <string, object> {
{ "subject", "calibration.add_ref_data" }, { "ref_data", _calibrationData.ToArray() }
});
_calibrationData.Clear();
if (_currCalibPoint >= _calibPoints.Length)
{
StopCalibration();
}
}
}
}
void NetMQClient()
{
AsyncIO.ForceDotNet.Force();
NetMQConfig.ManualTerminationTakeOver();
NetMQConfig.ContextCreate(true);
List <string> subports = new List <string>(); // subports for each client connection
// loop through all clients and try to connect to them
foreach (PupilConfiguration.PupilClient c in clients)
{
string subport = "";
string IPHeader = ">tcp://" + c.ip + ":";
bool frameReceived = false;
Debug.LogFormat("Requesting socket for {0}:{1} ({2})", c.ip, c.port, c.name);
RequestSocket requestSocket;
try
{
// validate ip header
if (!validateIPHeader(c.ip, c.port))
{
Debug.LogWarningFormat("{0}:{1} is not a valid ip header for client {2}", c.ip, c.port, c.name);
string failHeader = "";
subports.Add(failHeader);
IPHeaders.Add(failHeader);
c.is_connected = false;
continue;
}
requestSocket = new RequestSocket(IPHeader + c.port);
if (requestSocket != null)
{
requestSocket.SendFrame("SUB_PORT");
timeout = new System.TimeSpan(0, 0, 0, 100);
frameReceived = requestSocket.TryReceiveFrameString(timeout, out subport); // request subport, will be saved in var subport for this client
if (frameReceived)
{
if (c.initially_active)
{
subports.Add(subport);
IPHeaders.Add(IPHeader);
c.is_connected = true;
}
else
{
string failHeader = "";
subports.Add(failHeader);
IPHeaders.Add(failHeader);
c.is_connected = false;
Debug.LogWarningFormat("Skipped connection to client {0}:{1} ({2})", c.ip, c.port, c.name);
}
}
else
{
string failHeader = "";
subports.Add(failHeader);
IPHeaders.Add(failHeader);
c.is_connected = false;
Debug.LogWarningFormat("Could not connect to client {0}:{1} ({2}). Make sure address is corect and pupil remote service is running", c.ip, c.port, c.name);
}
requestSockets.Add(requestSocket);
}
}
catch (Exception e)
{
Debug.LogWarningFormat("Could not reach to client {0}:{1} ({2}): {4}", c.ip, c.port, c.name, e.ToString());
}
}
isConnected = true; // check if all clients are connected
if (isConnected)
{
Debug.LogFormat("Connected to {0} sockets", IPHeaders.Count);
foreach (String header in IPHeaders)
{
if (header.Equals(""))
{
subscriberSockets.Add(new SubscriberSocket());
continue;
}
else
{
SubscriberSocket subscriberSocket = new SubscriberSocket(header + subports[IPHeaders.IndexOf(header)]);
if (clients[IPHeaders.IndexOf(header)].detect_surface)
{
subscriberSocket.Subscribe("surface");
}
subscriberSocket.Subscribe("pupil.");
subscriberSocket.Subscribe("notify.");
//subscriberSocket.Subscribe("calibration.");
//subscriberSocket.Subscribe("logging.info");
//subscriberSocket.Subscribe("calibration_routines.calibrate");
//subscriberSocket.Subscribe("frame.");
//subscriberSocket.Subscribe("gaze.");
subscriberSockets.Add(subscriberSocket);
}
}
var msg = new NetMQMessage();
turn = 0; // used receive a message from each client in turn
while (!stop_thread_)
{
if (IPHeaders.Count != clients.Count)
{
break;
}
turn = ++turn % IPHeaders.Count;
if (IPHeaders[turn].Equals("") || clients[turn].is_connected == false)
{
continue;
}
timeout = new System.TimeSpan(0, 0, 0, 0, 1); // wait 200ms to receive a message
bool stillAlive = subscriberSockets[turn].TryReceiveMultipartMessage(timeout, ref (msg));
if (stillAlive)
{
try
{
string msgType = msg[0].ConvertToString();
var message = MsgPack.Unpacking.UnpackObject(msg[1].ToByteArray());
MsgPack.MessagePackObject mmap = message.Value;
if (msgType.Contains("pupil"))
{
// pupil detected
lock (thisLock_)
{
pupilData = JsonUtility.FromJson <Pupil.PupilData3D>(mmap.ToString());
}
}
if (msgType.Contains("frame"))
{
}
if (msgType.Contains("gaze"))
{
}
if (msgType.Contains("surfaces"))
{
// surface detected
lock (thisLock_)
{
if (!newData)
{
newData = true;
surfaceData = JsonUtility.FromJson <Pupil.SurfaceData3D>(mmap.ToString());
currentClient = clients[turn];
}
}
}
if (msgType.Equals("notify.calibration.started"))
{
//Debug.LogFormat("Calibration for client {0} started: {1}", clients[turn].name, mmap.ToString());
}
if (msgType.Equals("notify.calibration.failed"))
{
calibrationDoneClient = clients[turn];
//Debug.LogFormat("Calibration for client {0} failed", clients[turn].name);
}
if (msgType.Equals("notify.calibration.successful"))
{
clients[turn].is_calibrated = true;
calibrationDoneClient = clients[turn];
//Debug.LogFormat("Calibration for client {0} successful", clients[turn].name);
}
if (msgType.Equals("notify.calibration.calibration_data"))
{
//Debug.LogFormat("New calibration data for client {0}: {1}", clients[turn].name, mmap.ToString());
}
if (msgType.Equals("logging.info"))
{
//Debug.LogFormat("logging info for client {0}: {1}", clients[turn].name, mmap.ToString());
}
if (msgType.Equals("calibration_routines.calibrate"))
{
//Debug.LogFormat("Calibration info for client {0}: {1}", clients[turn].name, mmap.ToString());
}
}
catch
{
Debug.LogWarningFormat("Failed to deserialize pupil data for client {0}", clients[turn].name);
}
}
}
foreach (SubscriberSocket s in subscriberSockets)
{
s.Close();
}
subscriberSockets.Clear();
}
else
{
Debug.LogWarning("Failed to connect to all clients specified in config file");
}
NetMQConfig.ContextTerminate();
}
// Client thread which does not block Update()
void NetMQClient()
{
string IPHeader = ">tcp://" + IP + ":";
var timeout = new System.TimeSpan(0, 0, 1); //1sec
// Necessary to handle this NetMQ issue on Unity editor
// https://github.com/zeromq/netmq/issues/526
AsyncIO.ForceDotNet.Force();
NetMQConfig.ManualTerminationTakeOver();
NetMQConfig.ContextCreate(true);
string subport = "";
Debug.Log("Connect to the server: " + IPHeader + PORT + ".");
var requestSocket = new RequestSocket(IPHeader + PORT);
double t = 0;
const int N = 1000;
bool is_connected = false;
for (int k = 0; k < N; k++)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
requestSocket.SendFrame("SUB_PORT");
is_connected = requestSocket.TryReceiveFrameString(timeout, out subport);
sw.Stop();
t = t + sw.Elapsed.Milliseconds;
//Debug.Log("Round trip time:" + sw.Elapsed + "[sec].");
if (is_connected == false)
{
break;
}
}
Debug.Log("Round trip average time:" + t / N + "[msec].");
requestSocket.Close();
if (is_connected)
{
//
var subscriberSocket = new SubscriberSocket(IPHeader + subport);
subscriberSocket.Subscribe(ID);
var msg = new NetMQMessage();
while (is_connected && stop_thread_ == false)
{
Debug.Log("Receive a multipart message.");
is_connected = subscriberSocket.TryReceiveMultipartMessage(timeout, ref (msg));
if (is_connected)
{
Debug.Log("Unpack a received multipart message.");
try
{
//Debug.Log(msg[0].ConvertToString());
var message = MsgPack.Unpacking.UnpackObject(msg[1].ToByteArray());
MsgPack.MessagePackObject mmap = message.Value;
lock (thisLock_)
{
data_ = JsonUtility.FromJson <Pupil.PupilData3D>(mmap.ToString());
}
//Debug.Log(message);
}
catch
{
Debug.Log("Failed to unpack.");
}
}
else
{
Debug.Log("Failed to receive a message.");
Thread.Sleep(1000);
}
}
subscriberSocket.Close();
}
else
{
Debug.Log("Failed to connect the server.");
}
// Necessary to handle this NetMQ issue on Unity editor
// https://github.com/zeromq/netmq/issues/526
Debug.Log("ContextTerminate.");
NetMQConfig.ContextTerminate();
}
