private features[] toBinaryFeatures(engagement[] nonBinaryFeats, int nTotalBodies)
{
features[] binaryFeats = new features[nTotalBodies];
for (int i = 0; i < nTotalBodies; i++)
{
binaryFeats[i].color = nonBinaryFeats[i].color;
// (kinect engagement value, for debug and test purposes (not a binary feat for the model)
if (nonBinaryFeats[i].engaged > BINARY_ENG)
binaryFeats[i].engaged = 1;
else
binaryFeats[i].engaged = 0;
// TALKING
if (nonBinaryFeats[i].sound > BINARY_TALKING)
binaryFeats[i].talking = 1;
else
binaryFeats[i].talking = 0;
// SMILING
if (nonBinaryFeats[i].happy > BINARY_SMILE)
binaryFeats[i].smiling = 1;
else
binaryFeats[i].smiling = 0;
// LEAN BACK/FORWARD
if (nonBinaryFeats[i].leany > BINARY_LEANFORWARD)
binaryFeats[i].leanforward = 1;
else
binaryFeats[i].leanforward = 0;
if (nonBinaryFeats[i].leany < BINARY_LEANBACKWARD && Features[i].bodycount > 0)
binaryFeats[i].leanback = 1;
else
binaryFeats[i].leanback = 0;
// LOOK UP/DOWN
if (nonBinaryFeats[i].pitch > BINARY_LOOKUP)
binaryFeats[i].lookUp = 1;
else
binaryFeats[i].lookUp = 0;
if (nonBinaryFeats[i].pitch < BINARY_DOWN)
binaryFeats[i].lookDown = 1;
else
binaryFeats[i].lookDown = 0;
// LOOK AT ROBOTS
if (nonBinaryFeats[i].lookingAway > BINARY_LOOKAWAY || nonBinaryFeats[i].lookingAway == -1)
binaryFeats[i].lookRobots = 0;
else
binaryFeats[i].lookRobots = 1;
}
return binaryFeats;
}
private void printEngagementValues(engagement features, int partID)
{
Console.WriteLine("Color: " + features.color +
"\t engaged: " + features.engaged +
"\t smile:" + features.happy +
"\t looking away:" + features.lookingAway +
"\t leanY:" + features.leany +
"\t talking: " + features.sound +
"\t pitch: " + features.pitch +
"\t Z: " + features.Z +
"\t X: " + features.posx +
"\t Part: " + partID);
}
/// <summary>
/// Called every N miliseconds. Creates an engagement array of average engagement values in the past second
/// Resets engage array to 0 for collecting data in the next second
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
private void onTimer(object source, ElapsedEventArgs e)
{
engagement[] averages = new engagement[this.bodyCount];
for (int i = 0; i < this.bodyCount; i++)
{
averages[i].color = Features[i].color;
averages[i].facecount = Features[i].facecount;
averages[i].bodycount = Features[i].bodycount;
averages[i].soundcount = Features[i].soundcount;
averages[i].Z = Features[i].Z;
if (Features[i].facecount != 0)
{
averages[i].happy = Features[i].happy / Features[i].facecount;
averages[i].engaged = Features[i].engaged / Features[i].facecount;
averages[i].lookingAway = Features[i].lookingAway / Features[i].facecount;
averages[i].pitch = Features[i].pitch / Features[i].facecount;
}
else
{
averages[i].happy = -1;
averages[i].engaged = -1;
averages[i].lookingAway = -1;
}
if (Features[i].bodycount != 0)
{
averages[i].leanx = Features[i].leanx / Features[i].bodycount;
averages[i].leany = Features[i].leany / Features[i].bodycount;
averages[i].posx = Features[i].posx; //always saves the latest X
}
else
{
averages[i].leanx = -1; //should still check if bodycount is 0, because -1 is within range of lean values
averages[i].leany = -1;
averages[i].posx = -1;
}
averages[i].sound = Features[i].sound;
}
features[] binaryFeats = new features[this.bodyCount];
if (EXP_MODE)
{
engagement[] threeBodies = getTop3Bodies(averages, this.bodyCount);
if (threeBodies == null)
{
this.setFeaturesToZero();
return;
}
else
{
binaryFeats = new features[MAXKIDS];
binaryFeats = this.toBinaryFeatures(threeBodies, MAXKIDS);
}
}
else
{
Console.WriteLine("sending all feature vectors!");
binaryFeats = new features[this.bodyCount];
binaryFeats = this.toBinaryFeatures(averages, this.bodyCount); //change to MAXKIDS later!
}
for (int binaryIndex = 0; binaryIndex < MAXKIDS; binaryIndex++ )
{
for (int averagesIndex = 0; averagesIndex < this.bodyCount; averagesIndex++)
{
if (binaryFeats[binaryIndex].color.Equals(averages[averagesIndex].color))
{
printEnagagementAndFeatures(averages[averagesIndex], binaryFeats[binaryIndex]);
break;
}
}
}
/* for (int i = 0; i < this.bodyCount; i++)
{
//just to print only the tracked bodies
if (Features[i].bodycount > 0)
{
printEngagementValues(averages[i], i);
// printBinaryFeatureValues(binaryFeats[i], i);
// Console.WriteLine("");
}
}*/
String stringOut = serializeEngagement(binaryFeats);
if (sender.ConnectionEstablished)
{
//Console.WriteLine("sending: " + stringOut);
sender.Send(stringOut);
}
else
{
Console.WriteLine("socket no longer connected!");
}
this.setFeaturesToZero();
}
private void printEnagagementAndFeatures(engagement averageFeats, features binaryFeats)
{
Console.WriteLine(averageFeats.color + /* " (" + binaryFeats.color + ")" +*/
"\t Talk: " + averageFeats.sound.ToString("0.00") + "(" + binaryFeats.talking + ")" +
"\t Smile:" + averageFeats.happy.ToString("0.00") + "(" + binaryFeats.smiling + ")" +
"\t Y:" + averageFeats.leany.ToString("0.00") + " LeanForward(" + binaryFeats.leanforward + ")" + " LeanBack(" + binaryFeats.leanback + ")" +
"\t pitch: " + averageFeats.pitch.ToString("0.00") + " LookUp(" + binaryFeats.lookUp + ")" + " LookDown(" + binaryFeats.lookDown + ")" +
"\t looking away:" + averageFeats.lookingAway +
"\t Z: " + averageFeats.Z.ToString("0.00") +
"\t X: " + averageFeats.posx);
}
/// <summary>
/// ongoing: creates a list with only the active bodies; sort by Z, select the first three.
/// In the final three bodies, sort by X so that we always have each body the same order
///
/// </summary>
private engagement[] getTop3Bodies(engagement[] features, int nTotalBodies)
{
engagement[] top3Bodies = new engagement[MAXKIDS];
List<engagement> tmpBodies = new List<engagement>();
//create a list with all eligible bodies (body and face count larger than 2)
for (int i = 0; i < nTotalBodies; i++)
{
if (features[i].facecount > 2 && features[i].bodycount > 2)
{
tmpBodies.Add(features[i]);
}
}
if (tmpBodies.Count < MAXKIDS)
{
//Console.WriteLine("LESS THAN 3 KIDS ON THE SCENE!");
return null;
}
List<engagement> orderedListByZ = tmpBodies.OrderBy(x => x.Z).ToList();
//if more than 3, select the closest 3
if (orderedListByZ.Count > MAXKIDS)
{
Console.WriteLine("WARNING! more than 3 bodies, selecting the 3 closest ones");
tmpBodies = new List<engagement>();
for (int i = 0; i < MAXKIDS; i++)
{
tmpBodies.Add(orderedListByZ[i]);
}
}
List<engagement> orderedByX = tmpBodies.OrderBy(x => x.posx).ToList();
for (int i = 0; i < MAXKIDS; i++)
{
top3Bodies[i] = orderedByX[i];
// Console.Write("adding body... "); // + top3Bodies[i].color);
//printEngagementValues(top3Bodies[i], i);
}
return top3Bodies;
}
