private features[] toBinaryFeatures(engagement[] nonBinaryFeats, int nTotalBodies) { features[] binaryFeats = new features[nTotalBodies]; for (int i = 0; i < nTotalBodies; i++) { binaryFeats[i].color = nonBinaryFeats[i].color; // (kinect engagement value, for debug and test purposes (not a binary feat for the model) if (nonBinaryFeats[i].engaged > BINARY_ENG) binaryFeats[i].engaged = 1; else binaryFeats[i].engaged = 0; // TALKING if (nonBinaryFeats[i].sound > BINARY_TALKING) binaryFeats[i].talking = 1; else binaryFeats[i].talking = 0; // SMILING if (nonBinaryFeats[i].happy > BINARY_SMILE) binaryFeats[i].smiling = 1; else binaryFeats[i].smiling = 0; // LEAN BACK/FORWARD if (nonBinaryFeats[i].leany > BINARY_LEANFORWARD) binaryFeats[i].leanforward = 1; else binaryFeats[i].leanforward = 0; if (nonBinaryFeats[i].leany < BINARY_LEANBACKWARD && Features[i].bodycount > 0) binaryFeats[i].leanback = 1; else binaryFeats[i].leanback = 0; // LOOK UP/DOWN if (nonBinaryFeats[i].pitch > BINARY_LOOKUP) binaryFeats[i].lookUp = 1; else binaryFeats[i].lookUp = 0; if (nonBinaryFeats[i].pitch < BINARY_DOWN) binaryFeats[i].lookDown = 1; else binaryFeats[i].lookDown = 0; // LOOK AT ROBOTS if (nonBinaryFeats[i].lookingAway > BINARY_LOOKAWAY || nonBinaryFeats[i].lookingAway == -1) binaryFeats[i].lookRobots = 0; else binaryFeats[i].lookRobots = 1; } return binaryFeats; }
private void printEngagementValues(engagement features, int partID) { Console.WriteLine("Color: " + features.color + "\t engaged: " + features.engaged + "\t smile:" + features.happy + "\t looking away:" + features.lookingAway + "\t leanY:" + features.leany + "\t talking: " + features.sound + "\t pitch: " + features.pitch + "\t Z: " + features.Z + "\t X: " + features.posx + "\t Part: " + partID); }
/// <summary> /// Called every N miliseconds. Creates an engagement array of average engagement values in the past second /// Resets engage array to 0 for collecting data in the next second /// </summary> /// <param name="source"></param> /// <param name="e"></param> private void onTimer(object source, ElapsedEventArgs e) { engagement[] averages = new engagement[this.bodyCount]; for (int i = 0; i < this.bodyCount; i++) { averages[i].color = Features[i].color; averages[i].facecount = Features[i].facecount; averages[i].bodycount = Features[i].bodycount; averages[i].soundcount = Features[i].soundcount; averages[i].Z = Features[i].Z; if (Features[i].facecount != 0) { averages[i].happy = Features[i].happy / Features[i].facecount; averages[i].engaged = Features[i].engaged / Features[i].facecount; averages[i].lookingAway = Features[i].lookingAway / Features[i].facecount; averages[i].pitch = Features[i].pitch / Features[i].facecount; } else { averages[i].happy = -1; averages[i].engaged = -1; averages[i].lookingAway = -1; } if (Features[i].bodycount != 0) { averages[i].leanx = Features[i].leanx / Features[i].bodycount; averages[i].leany = Features[i].leany / Features[i].bodycount; averages[i].posx = Features[i].posx; //always saves the latest X } else { averages[i].leanx = -1; //should still check if bodycount is 0, because -1 is within range of lean values averages[i].leany = -1; averages[i].posx = -1; } averages[i].sound = Features[i].sound; } features[] binaryFeats = new features[this.bodyCount]; if (EXP_MODE) { engagement[] threeBodies = getTop3Bodies(averages, this.bodyCount); if (threeBodies == null) { this.setFeaturesToZero(); return; } else { binaryFeats = new features[MAXKIDS]; binaryFeats = this.toBinaryFeatures(threeBodies, MAXKIDS); } } else { Console.WriteLine("sending all feature vectors!"); binaryFeats = new features[this.bodyCount]; binaryFeats = this.toBinaryFeatures(averages, this.bodyCount); //change to MAXKIDS later! } for (int binaryIndex = 0; binaryIndex < MAXKIDS; binaryIndex++ ) { for (int averagesIndex = 0; averagesIndex < this.bodyCount; averagesIndex++) { if (binaryFeats[binaryIndex].color.Equals(averages[averagesIndex].color)) { printEnagagementAndFeatures(averages[averagesIndex], binaryFeats[binaryIndex]); break; } } } /* for (int i = 0; i < this.bodyCount; i++) { //just to print only the tracked bodies if (Features[i].bodycount > 0) { printEngagementValues(averages[i], i); // printBinaryFeatureValues(binaryFeats[i], i); // Console.WriteLine(""); } }*/ String stringOut = serializeEngagement(binaryFeats); if (sender.ConnectionEstablished) { //Console.WriteLine("sending: " + stringOut); sender.Send(stringOut); } else { Console.WriteLine("socket no longer connected!"); } this.setFeaturesToZero(); }
private void printEnagagementAndFeatures(engagement averageFeats, features binaryFeats) { Console.WriteLine(averageFeats.color + /* " (" + binaryFeats.color + ")" +*/ "\t Talk: " + averageFeats.sound.ToString("0.00") + "(" + binaryFeats.talking + ")" + "\t Smile:" + averageFeats.happy.ToString("0.00") + "(" + binaryFeats.smiling + ")" + "\t Y:" + averageFeats.leany.ToString("0.00") + " LeanForward(" + binaryFeats.leanforward + ")" + " LeanBack(" + binaryFeats.leanback + ")" + "\t pitch: " + averageFeats.pitch.ToString("0.00") + " LookUp(" + binaryFeats.lookUp + ")" + " LookDown(" + binaryFeats.lookDown + ")" + "\t looking away:" + averageFeats.lookingAway + "\t Z: " + averageFeats.Z.ToString("0.00") + "\t X: " + averageFeats.posx); }
/// <summary> /// ongoing: creates a list with only the active bodies; sort by Z, select the first three. /// In the final three bodies, sort by X so that we always have each body the same order /// /// </summary> private engagement[] getTop3Bodies(engagement[] features, int nTotalBodies) { engagement[] top3Bodies = new engagement[MAXKIDS]; List<engagement> tmpBodies = new List<engagement>(); //create a list with all eligible bodies (body and face count larger than 2) for (int i = 0; i < nTotalBodies; i++) { if (features[i].facecount > 2 && features[i].bodycount > 2) { tmpBodies.Add(features[i]); } } if (tmpBodies.Count < MAXKIDS) { //Console.WriteLine("LESS THAN 3 KIDS ON THE SCENE!"); return null; } List<engagement> orderedListByZ = tmpBodies.OrderBy(x => x.Z).ToList(); //if more than 3, select the closest 3 if (orderedListByZ.Count > MAXKIDS) { Console.WriteLine("WARNING! more than 3 bodies, selecting the 3 closest ones"); tmpBodies = new List<engagement>(); for (int i = 0; i < MAXKIDS; i++) { tmpBodies.Add(orderedListByZ[i]); } } List<engagement> orderedByX = tmpBodies.OrderBy(x => x.posx).ToList(); for (int i = 0; i < MAXKIDS; i++) { top3Bodies[i] = orderedByX[i]; // Console.Write("adding body... "); // + top3Bodies[i].color); //printEngagementValues(top3Bodies[i], i); } return top3Bodies; }