public Form1(string characterName)
{
InitializeComponent();
mKinectData = new KinectData();
mKinectData2 = new KinectData(); //second user data
GazeOut1=new GazeDef();
GazeOut2 = new GazeDef();
GazeOut1 = GazeDef.none;
GazeOut2 = GazeDef.none;
LastGazeOut1 = new GazeDef();
LastGazeOut2 = new GazeDef();
LastGazeOut1 = GazeDef.none;
LastGazeOut2 = GazeDef.none;
simpleSound = new SoundPlayer(@"countA.wav");
eyebrowControllerUsr1 = new EyebrowsController();
eyebrowControllerUsr2 = new EyebrowsController();
serialPort1 = new System.IO.Ports.SerialPort();
serialPort2 = new System.IO.Ports.SerialPort();
sentonce = false;
serialPort1.DataReceived += new System.IO.Ports.SerialDataReceivedEventHandler(serialPort1_DataReceived);
serialPort2.DataReceived += new System.IO.Ports.SerialDataReceivedEventHandler(serialPort2_DataReceived);
thalamusClient = new PerceptionClient(characterName);
pipeServer = new PipeServer.Server();
engagementpipe = new PipeServer.Server();
Xp = 0;
Yp = 0;
Zp = 0;
Xp2 = -1;
Yp2 = -1;
Zp2 = -1;
Xpold = 0;
Ypold = 0;
Zpold = 0;
Xp2old = -1;
Yp2old = -1;
Zp2old = -1;
RotH = 0;
RotV = 0;
RotH2 = 0;
RotV2 = 0;
depth = 0;
allOKAOdata = "";
allOKAOdata2 = "";
allQdata = "";
allQdata2 = "";
allkinectdata = "";
allkinectdata2 = "";
startstop = false;
FparticipanID = 0;
FparticipanID2 = 1;
Fparticipantname = "None";
Fparticipantname2 = "None2";
isEmpathic = true; //default value
OKAOdata = "";
OKAOdata2 = "";
Kinectdata = "";
Kinectdata2 = "";
Qdata1 = "";
Qdata2 = "";
saver = new System.Timers.Timer();
saver.Elapsed += new ElapsedEventHandler(OnsaverEvent);
saver.Enabled = false;
saver.Interval = 250; //Herz
DetectedPerson = false;
aTimer = new System.Timers.Timer();
aTimer.Stop();
aTimer.Elapsed += new ElapsedEventHandler(OnTimedEvent); //publisher
aTimer.Interval = 350;//increased the timer to reduce thalamus messages
videoTimer = new MicroLibrary.MicroTimer();
videoTimer.Stop();
videoTimer.MicroTimerElapsed += new MicroLibrary.MicroTimer.MicroTimerElapsedEventHandler(saveframe);
videoTimer.Interval = 33333; //30fps
//thalamusClient.ClientConnected += connected;
this.pipeServer.MessageReceived +=
new PipeServer.Server.MessageReceivedHandler(pipeServer_MessageReceived);
this.engagementpipe.MessageReceived +=
new PipeServer.Server.MessageReceivedHandler(engagementpipe_MessageReceived);
button1_Click(this,null); //auto start the pipe servers
if (scenarioselected == 0)
scenarioselected = 1;
}
void OnTimedEvent(object source, ElapsedEventArgs e) //thalamus publisher
{
#region Kinect2
if (kinectenable.Checked==true)
{
//pointing and mutual pointing publisher
if (scenarioselected==2)
{
if (lefthandX_1 >= -0.58 && lefthandX_1 <= 0.58 && lefthandY_1 <= -0.32 && lefthandZ_1 <= 0.65) //left hand of user1 above the screen area
{
thalamusClient.PerceptionPublisher.PointingPosition(1, EmoteCommonMessages.Hand.lefthand, lefthandX_1, lefthandY_1, lefthandZ_1);
//now check for mutual point with user2 first with user2 lefthand
if ((lefthandX_1 >= lefthandX_2 - 0.15) && (lefthandX_1 <= lefthandX_2 + 0.15) && (lefthandZ_1 >= lefthandZ_2 - 0.15) && (lefthandZ_1 <= lefthandZ_2 + 0.15))
thalamusClient.PerceptionPublisher.UserMutualPoint(true, ((lefthandX_1 + lefthandX_2) / 2), ((lefthandZ_1 + lefthandZ_2) / 2));
//then with user2 righthand
if ((lefthandX_1 >= righthandX_2 - 0.15) && (lefthandX_1 <= righthandX_2 + 0.15) && (lefthandZ_1 >= righthandZ_2 - 0.15) && (lefthandZ_1 <= righthandZ_2 + 0.15))
thalamusClient.PerceptionPublisher.UserMutualPoint(true, ((lefthandX_1 + righthandX_2) / 2), ((lefthandZ_1 + righthandZ_2) / 2));
}
if (righthandX_1 >= -0.58 && righthandX_1 <= 0.58 && righthandY_1 <= -0.32 && righthandZ_1 <= 0.65) //right hand of user1 above the screen area
{
thalamusClient.PerceptionPublisher.PointingPosition(1, EmoteCommonMessages.Hand.righthand, righthandX_1, righthandY_1, righthandZ_1);
//now check for mutual point with user2 first with user2 lefthand
if ((righthandX_1 >= lefthandX_2 - 0.15) && (righthandX_1 <= lefthandX_2 + 0.15) && (righthandZ_1 >= lefthandZ_2 - 0.15) && (righthandZ_1 <= lefthandZ_2 + 0.15))
thalamusClient.PerceptionPublisher.UserMutualPoint(true, ((righthandX_1 + lefthandX_2) / 2), ((righthandZ_1 + lefthandZ_2) / 2));
//then with user2 righthand
if ((righthandX_1 >= righthandX_2 - 0.15) && (righthandX_1 <= righthandX_2 + 0.15) && (righthandZ_1 >= righthandZ_2 - 0.15) && (righthandZ_1 <= righthandZ_2 + 0.15))
thalamusClient.PerceptionPublisher.UserMutualPoint(true, ((righthandX_1 + righthandX_2) / 2), ((righthandZ_1 + righthandZ_2) / 2));
}
if (lefthandX_2 >= -0.58 && lefthandX_2 <= 0.58 && lefthandY_2 <= -0.32 && lefthandZ_2 <= 0.65) //left hand of user2 above the screen area
{
thalamusClient.PerceptionPublisher.PointingPosition(2, EmoteCommonMessages.Hand.lefthand, lefthandX_2, lefthandY_2, lefthandZ_2);
}
if (righthandX_2 >= -0.58 && righthandX_2 <= 0.58 && righthandY_2 <= -0.32 && righthandZ_2 <= 0.65) //right hand of user2 above the screen area
{
thalamusClient.PerceptionPublisher.PointingPosition(2, EmoteCommonMessages.Hand.righthand, righthandX_2, righthandY_2, righthandZ_2);
}
//eye brow publisher
//it utilises the same controller. It uses au2 for left eye and au4 for right eye
eyebrowControllerUsr1.SetBrowsValues(eyebrowleft_1, eyebrowright_1);
if (scenarioselected == 2)
{
eyebrowControllerUsr2.SetBrowsValues(eyebrowleft_2, eyebrowright_2);
}
thalamusClient.PerceptionPublisher.EyebrowsAU(eyebrowControllerUsr1.AU2BrowRaiser, eyebrowControllerUsr1.AU4BrowLower, eyebrowControllerUsr2.AU2BrowRaiser, eyebrowControllerUsr2.AU4BrowLower); //old method
}
//detecting mutual gazing and publisher
if (rotY_1 < -40 && rotY_2 > 40)
if(gazed==false)
{
thalamusClient.PerceptionPublisher.UserMutualGaze(true);
gazed=true;
}
else
if(gazed==true)
{
thalamusClient.PerceptionPublisher.UserMutualGaze(false);
gazed = false;
}
//detecting touch chin and publisher
//first user both hands
if ((lefthandX_1 >= chinPosX_1 - 0.1 && lefthandX_1 <= chinPosX_1 + 0.1 && lefthandY_1 >= chinPosY_1 - 0.2 && lefthandY_1 <= chinPosY_1-0.05 && lefthandZ_1 >= chinPosZ_1 - 0.2 ) || (righthandX_1 >= chinPosX_1 - 0.1 && righthandX_1 <= chinPosX_1 + 0.1 && righthandY_1 >= chinPosY_1 - 0.2 && righthandY_1 <= chinPosY_1-0.05 && righthandZ_1 >= chinPosZ_1 - 0.2 ))//user1 touched chin
if(chin1touched==false)
{
thalamusClient.PerceptionPublisher.UserTouchChin(1, true);
chin1touched = true;
}
else
if (chin1touched == true)
{
thalamusClient.PerceptionPublisher.UserTouchChin(1, false);
chin1touched = false;
}
//second user both hands
if (scenarioselected==2)
{
if ((lefthandX_2 >= chinPosX_2 - 0.1 && lefthandX_2 <= chinPosX_2 + 0.1 && lefthandY_2 >= chinPosY_2 - 0.2 && lefthandY_2 <= chinPosY_2 - 0.05 && lefthandZ_2 >= chinPosZ_2 - 0.2) || (righthandX_2 >= chinPosX_2 - 0.1 && righthandX_2 <= chinPosX_2 + 0.1 && righthandY_2 >= chinPosY_2 - 0.2 && righthandY_2 <= chinPosY_2 - 0.05 && righthandZ_2 >= chinPosZ_2 - 0.2))//user1 touched chin
if (chin2touched == false)
{
thalamusClient.PerceptionPublisher.UserTouchChin(2, true);
chin2touched = true;
}
else
if (chin2touched == true)
{
thalamusClient.PerceptionPublisher.UserTouchChin(2, false);
chin1touched = false;
}
}
//keep same names for both kinects
Xp = locX_1;
Yp = locY_1;
Zp = locZ_1;
Xp2 = locX_2;
Yp2 = locY_2;
Zp2 = locZ_2;
}
#endregion
if (isEmpathic == true) //publish head position only if its empathic session
{
if (Xp != Xpold && Yp != Ypold && Zp != Zpold)
{
thalamusClient.PerceptionPublisher.HeadTracking(1, Xp, Yp, Zp, DetectedPerson);
Xpold = Xp;
Ypold = Yp;
Zpold = Zp;
}
if (Xp2 != Xp2old && Yp2 != Yp2old && Zp2 != Zp2old)
{
thalamusClient.PerceptionPublisher.HeadTracking(2, Xp2, Yp2, Zp2, DetectedPerson2);
Xp2old = Xp2;
Yp2old = Yp2;
Zp2old = Zp2;
}
//user 1 detected on screen form kinect
if (DetectedPerson == true)
{
if (GazeOut1 == GazeDef.screenL && GazeOut1 != LastGazeOut1) //screen left
thalamusClient.PerceptionPublisher.GazeTracking(1, EmoteCommonMessages.GazeEnum.ScreenLeft, GazeConfidence1);
if (GazeOut1 == GazeDef.screenR && GazeOut1 != LastGazeOut1) //screen right
thalamusClient.PerceptionPublisher.GazeTracking(1, EmoteCommonMessages.GazeEnum.ScreenRight, GazeConfidence1);
if (GazeOut1 == GazeDef.Robot && GazeOut1 != LastGazeOut1) //at robot
thalamusClient.PerceptionPublisher.GazeTracking(1, EmoteCommonMessages.GazeEnum.Robot, GazeConfidence1);
if (GazeOut1 == GazeDef.elsewhere && GazeOut1 != LastGazeOut1) //else
thalamusClient.PerceptionPublisher.GazeTracking(1, EmoteCommonMessages.GazeEnum.Other, GazeConfidence1);
if (GazeOut1 == GazeDef.none && GazeOut1 != LastGazeOut1) //none
thalamusClient.PerceptionPublisher.GazeTracking(1, EmoteCommonMessages.GazeEnum.None, GazeConfidence1);
LastGazeOut1 = GazeOut1;
//okao message publisher (latest known values)
thalamusClient.PerceptionPublisher.OKAOMessage(1, smile1, confidence1, o1Expression1, o1Expression2, o1Expression3, o1Expression4, o1Expression5, o1Expression6, o1Expression7, GazeOut1.ToString());
//q sensor publisher
if (Qdata1 != "")
{
string[] q_temp1 = Qdata1.Split(',');
thalamusClient.PerceptionPublisher.QSensorMessage(1, Double.Parse(q_temp1[0]), Double.Parse(q_temp1[1]), Double.Parse(q_temp1[2]), Double.Parse(q_temp1[3]), Double.Parse(q_temp1[4]));
}
}
//second user publishing only if in front of camera/kinect
if (DetectedPerson2 == true)
{
if (GazeOut2 == GazeDef.screenL && GazeOut2 != LastGazeOut2) //screen left
thalamusClient.PerceptionPublisher.GazeTracking(2, EmoteCommonMessages.GazeEnum.ScreenLeft, GazeConfidence2);
if (GazeOut2 == GazeDef.screenR && GazeOut2 != LastGazeOut2) //screen right
thalamusClient.PerceptionPublisher.GazeTracking(2, EmoteCommonMessages.GazeEnum.ScreenRight, GazeConfidence2);
if (GazeOut2 == GazeDef.Robot && GazeOut2 != LastGazeOut2) //at robot
thalamusClient.PerceptionPublisher.GazeTracking(2, EmoteCommonMessages.GazeEnum.Robot, GazeConfidence2);
if (GazeOut2 == GazeDef.elsewhere && GazeOut2 != LastGazeOut2) //else
thalamusClient.PerceptionPublisher.GazeTracking(2, EmoteCommonMessages.GazeEnum.Other, GazeConfidence2);
if (GazeOut1 == GazeDef.none && GazeOut2 != LastGazeOut2) //none
thalamusClient.PerceptionPublisher.GazeTracking(2, EmoteCommonMessages.GazeEnum.None, GazeConfidence2);
LastGazeOut2 = GazeOut2;
//okao message publisher (latest known values)
thalamusClient.PerceptionPublisher.OKAOMessage(2, smile2, confidence2, o2Expression1, o2Expression2, o2Expression3, o2Expression4, o2Expression5, o2Expression6, o2Expression7, GazeOut2.ToString());
//q sensor publisher
if (Qdata2 != "")
{
string[] q_temp2 = Qdata2.Split(',');
thalamusClient.PerceptionPublisher.QSensorMessage(2, Double.Parse(q_temp2[0]), Double.Parse(q_temp2[1]), Double.Parse(q_temp2[2]), Double.Parse(q_temp2[3]), Double.Parse(q_temp2[4]));
}
}
DetectedPerson = false;
DetectedPerson2 = false;
}
if (checkedListBox1.GetItemChecked(0)==true)//kinect v1 enabled
{
//publish mutual gaze similarly to v2
//detecting mutual gazing and publisher
if (scenarioselected == 2)
{
if (RotV < -40 && RotV2 > 40)
if (gazed == false)
{
thalamusClient.PerceptionPublisher.UserMutualGaze(true);
gazed = true;
}
else
if (gazed == true)
{
thalamusClient.PerceptionPublisher.UserMutualGaze(false);
gazed = false;
}
}
thalamusClient.PerceptionPublisher.EyebrowsAU(eyebrowControllerUsr1.AU2BrowRaiser, eyebrowControllerUsr1.AU4BrowLower, eyebrowControllerUsr2.AU2BrowRaiser, eyebrowControllerUsr2.AU4BrowLower);
}
}
void DisplayMessageReceived(PipeServer.Server.Client client, string message)
{
if (message.StartsWith("Temp")) //q sensor data
{
//no more active
}
if (message.Substring(1,3)=="Dep") //kinect1 data
{
Console.WriteLine("Kinect: " + message);
aTimer.Enabled = true;
if (message.StartsWith("1"))
{
string[] parseMsg = message.Split(';');
Kinectdata = message;
Kinectdata2 = "";
try
{
depth = Convert.ToDouble(parseMsg[0].Substring(parseMsg[0].IndexOf(':') + 1), ifp);
Xp = Convert.ToDouble(parseMsg[1].Substring(parseMsg[1].IndexOf(':') + 1), ifp);
Yp = Convert.ToDouble(parseMsg[2].Substring(parseMsg[2].IndexOf(':') + 1), ifp);
Zp = Convert.ToDouble(parseMsg[3].Substring(parseMsg[3].IndexOf(':') + 1), ifp);
RotH = Convert.ToDouble(parseMsg[4].Substring(parseMsg[4].IndexOf(':') + 1), ifp);
RotV = Convert.ToDouble(parseMsg[5].Substring(parseMsg[5].IndexOf(':') + 1), ifp);
}
catch
{
depth = 0;
Xp = 0;
Yp = 0;
Zp = 0;
RotH = 0;
RotV = 0;
}
DetectedPerson = true;
try
{
AU4BrowLower = Convert.ToDouble(parseMsg[10].Substring(parseMsg[10].IndexOf(':') + 1), ifp);
AU2BrowRaiser = Convert.ToDouble(parseMsg[11].Substring(parseMsg[11].IndexOf(':') + 1), ifp);
}
catch
{
AU4BrowLower = 0;
AU2BrowRaiser = 0;
}
eyebrowControllerUsr1.SetBrowsValues(AU2BrowRaiser, AU4BrowLower);
}
else
{
Kinectdata2 = message;
Kinectdata = "";
string[] parseMsg = message.Split(';');
try
{
depth = Convert.ToDouble(parseMsg[0].Substring(parseMsg[0].IndexOf(':') + 1), ifp);
Xp2 = Convert.ToDouble(parseMsg[1].Substring(parseMsg[1].IndexOf(':') + 1), ifp);
Yp2 = Convert.ToDouble(parseMsg[2].Substring(parseMsg[2].IndexOf(':') + 1), ifp);
Zp2 = Convert.ToDouble(parseMsg[3].Substring(parseMsg[3].IndexOf(':') + 1), ifp);
RotH2 = Convert.ToDouble(parseMsg[4].Substring(parseMsg[4].IndexOf(':') + 1), ifp);
RotV2 = Convert.ToDouble(parseMsg[5].Substring(parseMsg[5].IndexOf(':') + 1), ifp);
}
catch
{
depth = 0;
Xp2 = 0;
Yp2 = 0;
Zp2 = 0;
RotH2 = 0;
RotV2 = 0;
}
DetectedPerson2 = true;
try
{
AU4BrowLower = Convert.ToDouble(parseMsg[10].Substring(parseMsg[10].IndexOf(':') + 1), ifp);
AU2BrowRaiser = Convert.ToDouble(parseMsg[11].Substring(parseMsg[11].IndexOf(':') + 1), ifp);
}
catch
{
AU4BrowLower = 0;
AU2BrowRaiser = 0;
}
eyebrowControllerUsr2.SetBrowsValues(AU2BrowRaiser, AU4BrowLower);
}
checkedListBox1.SetItemChecked(0, true);
}
if (message.StartsWith("Cam")) //camera data
{
if (message.Substring(3, 1) == "1")
checkedListBox1.SetItemChecked(2, true);
if (message.Substring(3, 1) == "2")
checkedListBox1.SetItemChecked(3, true);
if (message.Substring(3, 1) == "3")
checkedListBox1.SetItemChecked(4, true);
if (message.Substring(3, 1) == "4")
checkedListBox1.SetItemChecked(5, true);
if (message.Substring(3, 1) == "5")
checkedListBox1.SetItemChecked(6, true);
if (message.Substring(3, 1) == "6")
checkedListBox1.SetItemChecked(7, true);
}
if (message.StartsWith("User")) //okao data
{
checkedListBox1.SetItemChecked(1, true);
OKAOdata = message;
string[] parseMsg1 = OKAOdata.Split(',');
if (Convert.ToDouble(parseMsg1[1]) == 1)
{//user 1
OKAOdata2 = "";
try
{
smile1 = Convert.ToInt16(parseMsg1[3]);
confidence1 = Convert.ToInt16(parseMsg1[2]);
fupdown1 = Convert.ToInt16(parseMsg1[11]);
fleftright1 = Convert.ToInt16(parseMsg1[12]);
o1Expression1= Convert.ToInt16(parseMsg1[4]);
o1Expression2= Convert.ToInt16(parseMsg1[5]);
o1Expression3= Convert.ToInt16(parseMsg1[6]);
o1Expression4= Convert.ToInt16(parseMsg1[7]);
o1Expression5= Convert.ToInt16(parseMsg1[8]);
o1Expression6 = Convert.ToInt16(parseMsg1[9]);
o1Expression7 = Convert.ToInt16(parseMsg1[10]);
if(parseMsg1[13]=="screenL")
GazeOut1=GazeDef.screenL;
if(parseMsg1[13]=="screenR")
GazeOut1=GazeDef.screenR;
if(parseMsg1[13]=="else")
GazeOut1=GazeDef.elsewhere;
if(parseMsg1[13]=="robot")
GazeOut1=GazeDef.Robot;
if (parseMsg1[13] == "none")
GazeOut1 = GazeDef.none;
this.Invoke(new Action(() =>{
lblgaze.Text = GazeOut1.ToString();
lblHeadX.Text = parseMsg1[11];
lblHeadY.Text = parseMsg1[12];
}
));
}
catch
{
smile1 = 0;
confidence1 = 0;
fupdown1 = 0;
fleftright1 = 0;
fupdown1 = 0;
fleftright1 = 0;
GazeOut1=GazeDef.none;
}
}
else
{//user 2
OKAOdata2 = message;
OKAOdata = "";
try
{
smile2 = Convert.ToInt16(parseMsg1[3]);
confidence2 = Convert.ToInt16(parseMsg1[2]);
fupdown2 = Convert.ToInt16(parseMsg1[11]);
fleftright2 = Convert.ToInt16(parseMsg1[12]);
o2Expression1 = Convert.ToInt16(parseMsg1[4]);
o2Expression2 = Convert.ToInt16(parseMsg1[5]);
o2Expression3 = Convert.ToInt16(parseMsg1[6]);
o2Expression4 = Convert.ToInt16(parseMsg1[7]);
o2Expression5 = Convert.ToInt16(parseMsg1[8]);
o2Expression6 = Convert.ToInt16(parseMsg1[9]);
o2Expression7 = Convert.ToInt16(parseMsg1[10]);
if (parseMsg1[13] == "screenL")
GazeOut2 = GazeDef.screenL;
if (parseMsg1[13] == "screenR")
GazeOut2 = GazeDef.screenR;
if (parseMsg1[13] == "else")
GazeOut2 = GazeDef.elsewhere;
if (parseMsg1[13] == "robot")
GazeOut2 = GazeDef.Robot;
if (parseMsg1[13] == "none")
GazeOut2 = GazeDef.none;
this.Invoke(new Action(() =>
{
lblgaze2.Text = GazeOut2.ToString();
lblHeadX2.Text = parseMsg1[11];
lblHeadY2.Text = parseMsg1[12];
}
));
}
catch
{
smile2 = 0;
confidence2 = 0;
fupdown2 = 0;
fleftright2 = 0;
GazeOut2 = GazeDef.none;
}
}
}
message = "";
}
