private features[] toBinaryFeatures(engagement[] nonBinaryFeats, int nTotalBodies)
{
features[] binaryFeats = new features[nTotalBodies];
for (int i = 0; i < nTotalBodies; i++)
{
binaryFeats[i].color = nonBinaryFeats[i].color;
// (kinect engagement value, for debug and test purposes (not a binary feat for the model)
if (nonBinaryFeats[i].engaged > BINARY_ENG)
binaryFeats[i].engaged = 1;
else
binaryFeats[i].engaged = 0;
// TALKING
if (nonBinaryFeats[i].sound > BINARY_TALKING)
binaryFeats[i].talking = 1;
else
binaryFeats[i].talking = 0;
// SMILING
if (nonBinaryFeats[i].happy > BINARY_SMILE)
binaryFeats[i].smiling = 1;
else
binaryFeats[i].smiling = 0;
// LEAN BACK/FORWARD
if (nonBinaryFeats[i].leany > BINARY_LEANFORWARD)
binaryFeats[i].leanforward = 1;
else
binaryFeats[i].leanforward = 0;
if (nonBinaryFeats[i].leany < BINARY_LEANBACKWARD && Features[i].bodycount > 0)
binaryFeats[i].leanback = 1;
else
binaryFeats[i].leanback = 0;
// LOOK UP/DOWN
if (nonBinaryFeats[i].pitch > BINARY_LOOKUP)
binaryFeats[i].lookUp = 1;
else
binaryFeats[i].lookUp = 0;
if (nonBinaryFeats[i].pitch < BINARY_DOWN)
binaryFeats[i].lookDown = 1;
else
binaryFeats[i].lookDown = 0;
// LOOK AT ROBOTS
if (nonBinaryFeats[i].lookingAway > BINARY_LOOKAWAY || nonBinaryFeats[i].lookingAway == -1)
binaryFeats[i].lookRobots = 0;
else
binaryFeats[i].lookRobots = 1;
}
return binaryFeats;
}
private void printEnagagementAndFeatures(engagement averageFeats, features binaryFeats)
{
Console.WriteLine(averageFeats.color + /* " (" + binaryFeats.color + ")" +*/
"\t Talk: " + averageFeats.sound.ToString("0.00") + "(" + binaryFeats.talking + ")" +
"\t Smile:" + averageFeats.happy.ToString("0.00") + "(" + binaryFeats.smiling + ")" +
"\t Y:" + averageFeats.leany.ToString("0.00") + " LeanForward(" + binaryFeats.leanforward + ")" + " LeanBack(" + binaryFeats.leanback + ")" +
"\t pitch: " + averageFeats.pitch.ToString("0.00") + " LookUp(" + binaryFeats.lookUp + ")" + " LookDown(" + binaryFeats.lookDown + ")" +
"\t looking away:" + averageFeats.lookingAway +
"\t Z: " + averageFeats.Z.ToString("0.00") +
"\t X: " + averageFeats.posx);
}
/// <summary>
/// Called every N miliseconds. Creates an engagement array of average engagement values in the past second
/// Resets engage array to 0 for collecting data in the next second
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
private void onTimer(object source, ElapsedEventArgs e)
{
engagement[] averages = new engagement[this.bodyCount];
for (int i = 0; i < this.bodyCount; i++)
{
averages[i].color = Features[i].color;
averages[i].facecount = Features[i].facecount;
averages[i].bodycount = Features[i].bodycount;
averages[i].soundcount = Features[i].soundcount;
averages[i].Z = Features[i].Z;
if (Features[i].facecount != 0)
{
averages[i].happy = Features[i].happy / Features[i].facecount;
averages[i].engaged = Features[i].engaged / Features[i].facecount;
averages[i].lookingAway = Features[i].lookingAway / Features[i].facecount;
averages[i].pitch = Features[i].pitch / Features[i].facecount;
}
else
{
averages[i].happy = -1;
averages[i].engaged = -1;
averages[i].lookingAway = -1;
}
if (Features[i].bodycount != 0)
{
averages[i].leanx = Features[i].leanx / Features[i].bodycount;
averages[i].leany = Features[i].leany / Features[i].bodycount;
averages[i].posx = Features[i].posx; //always saves the latest X
}
else
{
averages[i].leanx = -1; //should still check if bodycount is 0, because -1 is within range of lean values
averages[i].leany = -1;
averages[i].posx = -1;
}
averages[i].sound = Features[i].sound;
}
features[] binaryFeats = new features[this.bodyCount];
if (EXP_MODE)
{
engagement[] threeBodies = getTop3Bodies(averages, this.bodyCount);
if (threeBodies == null)
{
this.setFeaturesToZero();
return;
}
else
{
binaryFeats = new features[MAXKIDS];
binaryFeats = this.toBinaryFeatures(threeBodies, MAXKIDS);
}
}
else
{
Console.WriteLine("sending all feature vectors!");
binaryFeats = new features[this.bodyCount];
binaryFeats = this.toBinaryFeatures(averages, this.bodyCount); //change to MAXKIDS later!
}
for (int binaryIndex = 0; binaryIndex < MAXKIDS; binaryIndex++ )
{
for (int averagesIndex = 0; averagesIndex < this.bodyCount; averagesIndex++)
{
if (binaryFeats[binaryIndex].color.Equals(averages[averagesIndex].color))
{
printEnagagementAndFeatures(averages[averagesIndex], binaryFeats[binaryIndex]);
break;
}
}
}
/* for (int i = 0; i < this.bodyCount; i++)
{
//just to print only the tracked bodies
if (Features[i].bodycount > 0)
{
printEngagementValues(averages[i], i);
// printBinaryFeatureValues(binaryFeats[i], i);
// Console.WriteLine("");
}
}*/
String stringOut = serializeEngagement(binaryFeats);
if (sender.ConnectionEstablished)
{
//Console.WriteLine("sending: " + stringOut);
sender.Send(stringOut);
}
else
{
Console.WriteLine("socket no longer connected!");
}
this.setFeaturesToZero();
}
private void printBinaryFeatureValues(features features, int partID)
{
Console.WriteLine("Color: " + features.color +
"\t engaged: " + features.engaged +
"\t smile:" + features.smiling +
"\t looking robots:" + features.lookRobots +
"\t leanforward:" + features.leanforward +
"\t leanback:" + features.leanback +
"\t talking: " + features.talking +
"\t lookUp: " + features.lookUp +
"\t lookDown: " + features.lookDown);
}
/// <summary>
/// Convert an engagement array into a string/json to be sent through rosbridge
/// TO BE CHANGED
///
/// talking0:happ0:leanForward0:leanBack0:lookUp0:lookDown0:lookRobots0;talking1:happ1:leanForward1:leanBack1:lookUp1:lookDown1:lookRobots1;talking2 ...
///
/// </summary>
/// <returns> serialized String </returns>
public string serializeEngagement(features[] feats)
{
string outputData = "";
for (int i = 0; i < feats.Length; i++)
{
outputData += feats[i].engaged.ToString()
+ feats[i].talking.ToString()
+ feats[i].smiling.ToString()
+ feats[i].leanforward.ToString()
+ feats[i].leanback.ToString()
+ feats[i].lookUp.ToString()
+ feats[i].lookDown.ToString()
+ feats[i].lookRobots.ToString();
// do not write ":" on the last one
if (i != (feats.Length - 1))
outputData += ":";
}
//terminating character
outputData += '$';
Console.WriteLine("sending: " + feats[0].color + ", " + feats[1].color + ", " + feats[2].color);
return outputData;
}
