/** * Use this to average together confusion matrices * over different tests. */ public void AddResult(ConfusionMatrices other) { if (confusion_matrices.Count != other.confusion_matrices.Count) { throw new Exception("confusion_matrices.Count == other.confusion_matrices.Count"); } entries++; for (int ii = 0; ii < other.confusion_matrices.Count; ii++) { double sum = other.confusion_matrices[ii].Sum(); confusion_matrices[ii] += other.confusion_matrices[ii] / sum; } }
/** * Evaluate all user study sessions for a given device type. */ public static void evaluate_sessions(DeviceType device) { List <ConfusionMatrices> confusion_matrices = new List <ConfusionMatrices>(); int[] participants = get_participant_list(device); for (int i = 0; i < participants.Length - 1; i++) { ConfusionMatrices cm = evaluate_session( device, participants[i]); confusion_matrices.Add(cm); int idx = confusion_matrices.Count - 1; Console.WriteLine("Participant: " + participants[i]); ResultT resultss = confusion_matrices[idx].Results(); resultss.Print(); Console.WriteLine(); } // put all results into first confusion // matrix for (int ii = 1; ii < confusion_matrices.Count; ii++) { confusion_matrices[0].AddResult(confusion_matrices[ii]); } Console.WriteLine("Aggregate results:"); ResultT result = confusion_matrices[0].Results(); result.Print(); Console.WriteLine(); }
/** * After the command is complete, update the matrices * with detected gestures. */ public void update_confusion_matrices(ConfusionMatrices cm) { bool found = false; for (int ii = 0; ii < detected_ids.Count; ii++) { int detected_id = detected_ids[ii]; if (found && expected_id == detected_id) { // treat as false positive // because we've already detected // the gesture once cm.AddResult( expected_id, -2); continue; } cm.AddResult( expected_id, detected_id); // mark that we've found this gesture found = found || (detected_id == expected_id); } // false negative if (!found) { cm.AddResult( expected_id, -1); } }
/** * Load a participant's dataset and session. * Train the recognizer with the training data * and run the video through. See what happens... */ public static ConfusionMatrices evaluate_session(DeviceType device, int subject_id) { // Load up the training dataset. Dataset ds = load_subject_dataset( device, subject_id); // Load up the session. List <Frame> frames = new List <Frame>(); load_session( device, subject_id, frames); // Create a new recognizer. JackknifeBlades blades = new JackknifeBlades(); blades.SetIPDefaults(); Jackknife jk = new Jackknife(blades); // Train the recognizer, without 'bad' gestures. for (int ii = 0; ii < ds.Samples.Count; ii++) { int gesture_id = ds.Samples[ii].GestureId; string gesture_name = ds.Gestures[gesture_id]; if (bad_gesture(gesture_name)) { continue; } jk.AddTemplate(ds.Samples[ii]); } // Get device and application parameters // based on the device type. configuartion_parameters_t parameters = new configuartion_parameters_t(device); // We originally used n=4, r=2 for Kinect data // and n=6, r=2 for Leap Motion data, but // here we just set the average. There is barely // any effect on the results. jk.Train(6, 2, 1.00); // Play session video through // the recognizer. List <Vector> buffer = new List <Vector>(); List <int> detections = new List <int>(); List <CommandResults> cmds = new List <CommandResults>(); int last_cmd_id = -1; int next_update = parameters.update_interval; int frame_no = 0; ExponentialMovingAverage filter = new ExponentialMovingAverage(frames[0].pt); Vector pt; for (int ii = 0; ii < frames.Count; ii++) { // skip this frame if its bad if (frames[ii].bad_pt) { continue; } // Low pass filter the input. // Note, we originally didn't smooth the data, // so results now are a little higher than in // the paper. pt = filter.Filter( frames[ii].pt, 1 / (double)parameters.fps); //pt = frames[ii].pt; frame_no += 1; // start a new command if (frames[ii].cmd_id != last_cmd_id) { last_cmd_id = frames[ii].cmd_id; int gid = convert_gesture_id( ds, frames[ii].gesture_id); CommandResults cmd = new CommandResults( frames[ii].cmd_id, gid); if (bad_gesture(frames[ii].gesture_id)) { cmd.ignore = true; } cmds.Add(cmd); } // This buffering approach is really // inefficient, but since this off-line, // performance is not important. buffer.Add(pt); if (buffer.Count > parameters.sliding_window_frame_cnt) { buffer.RemoveAt(0); } // We need to have a couple points before // calling the recognizer. if (buffer.Count < 2) { continue; } // Wait a few frames again before trying // to recognize again. if (frame_no < next_update) { continue; } next_update = frame_no + parameters.update_interval; // Run the recognizer. int gesture_id = jk.Classify(buffer); // Add recognition result. detections.Add(gesture_id); if (detections.Count > parameters.repeat_cnt) { detections.RemoveAt(0); } // Count how many times this gesture was recognized. int winner_cnt = 0; for (int jj = 0; jj < detections.Count; jj++) { if (detections[jj] == gesture_id) { winner_cnt += 1; } } // Ensure we have enough recognitions. if (winner_cnt < parameters.repeat_cnt) { continue; } // If nothing was detected, skip rest. if (gesture_id == -1) { continue; } // Hurray! A gesture is recognized! // Hopefully it's the right one too!! cmds[cmds.Count - 1].add(gesture_id); detections.Clear(); buffer.Clear(); } // Mark bad commands, situations where the participant // made a mistake or tracking was lost. We know the // command was bad because the protector asked the // participant to repeat the gesture, but a new command // ID is assigned to the sequence. for (int ii = 1; ii < cmds.Count; ii++) { if (cmds[ii].expected_id == cmds[ii - 1].expected_id) { CommandResults temp = cmds[ii - 1]; temp.ignore = true; cmds[ii - 1] = temp; } } // Put all results in confusion matrices. ConfusionMatrices ret = new ConfusionMatrices(ds); for (int ii = 0; ii < cmds.Count; ii++) { if (cmds[ii].ignore) { continue; } cmds[ii].update_confusion_matrices(ret); } return(ret); }
/** * A simple user independent test. */ public static void user_indepedent_test(DeviceType device) { // First, load all training data for one device type. string path = ""; if (device == DeviceType.KINECT) { path = Global.GetRootDirectory() + "datasets/jk2017/kinect/training/"; } else if (device == DeviceType.LEAP_MOTION) { path = Global.GetRootDirectory() + "datasets/jk2017/leap_motion/training/"; } Dataset ds = Dataset.LoadDataset(path); int subject_cnt = ds.Subjects.Count; int sample_cnt = ds.Samples.Count; ConfusionMatrices cm = new ConfusionMatrices(ds); // iterate through all subjects for (int subject_id = 0; subject_id < subject_cnt; subject_id++) { ConfusionMatrices cm_individual = new ConfusionMatrices(ds); Console.WriteLine("Participant: " + subject_id); // train a recognizer with the selected subject JackknifeBlades blades = new JackknifeBlades(); blades.SetIPDefaults(); Jackknife jk = new Jackknife(blades); for (int sample_id = 0; sample_id < sample_cnt; sample_id++) { Sample sample = ds.Samples[sample_id]; if (sample.SubjectId != subject_id) { continue; } jk.AddTemplate(ds.Samples[sample_id]); } // only train the recognize if you need // test the recognizer with all other samples for (int sample_id = 0; sample_id < sample_cnt; sample_id++) { Sample sample = ds.Samples[sample_id]; if (sample.SubjectId == subject_id) { continue; } int gid = jk.Classify(sample); cm_individual.AddResult( sample.GestureId, gid); } cm.AddResult(cm_individual); ResultT resultss = cm_individual.Results(); resultss.Print(); } Console.WriteLine("Aggregate Results: "); ResultT results = cm.Results(); results.Print(); }