/**
* Train segmentor and recognizer with next participant's data
*/
void Prepare()
{
ResetParticipantsCommands();
GetNewParticipantSamples();
machete = new Machete(deviceType, cr_options);
foreach (Sample sample in trainSets[currentParticipantIndex])
{
machete.AddSample(sample, filtered: true);
}
blades = new JackknifeBlades();
blades.SetIPDefaults();
blades.ResampleCnt = Global.GetResampleCnt(deviceType);
blades.Radius = (int)(blades.ResampleCnt * .1f);
blades.LowerBound = false;
jackknife = new Jackknife.Jackknife(blades);
foreach (Jackknife.Sample sample in jkTrainSets[currentParticipantIndex])
{
jackknife.AddTemplate(sample);
}
jackknife.SetRejectionThresholds(knownThreshold);
frames = participantsFrames[currentParticipantIndex];
double cutoff = 3.0;
if (deviceType == DeviceType.MOUSE)
{
cutoff = 5.0;
}
ema_filter = new ExponentialMovingAverage(frames[0].pt, cutoff);
video = new List <Vector>();
rresults = new List <RecognitionResult>();
frame_idx = 0;
currentParticipantID = participants[currentParticipantIndex];
timer = 0.0;
}
void PrepareWindow()
{
ResetParticipantsCommands();
// Set up Jackknife first because window will use it's values
blades = new JackknifeBlades();
blades.SetIPDefaults();
blades.ResampleCnt = Global.GetResampleCnt(deviceType);
blades.Radius = (int)(blades.ResampleCnt * .1f);
blades.LowerBound = false;
jackknife = new Jackknife.Jackknife(blades);
foreach (Jackknife.Sample sample in jkTrainSets[currentParticipantIndex])
{
jackknife.AddTemplate(sample);
}
jackknife.SetRejectionThresholds(knownThreshold);
frames = participantsFrames[currentParticipantIndex];
double cutoff = 3.0;
if (deviceType == DeviceType.MOUSE)
{
cutoff = 5.0;
}
ema_filter = new ExponentialMovingAverage(frames[0].pt, cutoff);
video = new List <Vector>();
rresults = new List <RecognitionResult>();
frame_idx = 0;
currentParticipantID = participants[currentParticipantIndex];
window = new Window(jackknife, m: (int)windowMode);
timer = 0.0;
}
/**
* Load a participant's dataset and session.
* Train the recognizer with the training data
* and run the video through. See what happens...
*/
public static ConfusionMatrices evaluate_session(DeviceType device, int subject_id)
{
// Load up the training dataset.
Dataset ds = load_subject_dataset(
device,
subject_id);
// Load up the session.
List <Frame> frames = new List <Frame>();
load_session(
device,
subject_id,
frames);
// Create a new recognizer.
JackknifeBlades blades = new JackknifeBlades();
blades.SetIPDefaults();
Jackknife jk = new Jackknife(blades);
// Train the recognizer, without 'bad' gestures.
for (int ii = 0;
ii < ds.Samples.Count;
ii++)
{
int gesture_id = ds.Samples[ii].GestureId;
string gesture_name = ds.Gestures[gesture_id];
if (bad_gesture(gesture_name))
{
continue;
}
jk.AddTemplate(ds.Samples[ii]);
}
// Get device and application parameters
// based on the device type.
configuartion_parameters_t parameters = new configuartion_parameters_t(device);
// We originally used n=4, r=2 for Kinect data
// and n=6, r=2 for Leap Motion data, but
// here we just set the average. There is barely
// any effect on the results.
jk.Train(6, 2, 1.00);
// Play session video through
// the recognizer.
List <Vector> buffer = new List <Vector>();
List <int> detections = new List <int>();
List <CommandResults> cmds = new List <CommandResults>();
int last_cmd_id = -1;
int next_update = parameters.update_interval;
int frame_no = 0;
ExponentialMovingAverage filter = new ExponentialMovingAverage(frames[0].pt);
Vector pt;
for (int ii = 0;
ii < frames.Count;
ii++)
{
// skip this frame if its bad
if (frames[ii].bad_pt)
{
continue;
}
// Low pass filter the input.
// Note, we originally didn't smooth the data,
// so results now are a little higher than in
// the paper.
pt = filter.Filter(
frames[ii].pt,
1 / (double)parameters.fps);
//pt = frames[ii].pt;
frame_no += 1;
// start a new command
if (frames[ii].cmd_id != last_cmd_id)
{
last_cmd_id = frames[ii].cmd_id;
int gid = convert_gesture_id(
ds,
frames[ii].gesture_id);
CommandResults cmd = new CommandResults(
frames[ii].cmd_id,
gid);
if (bad_gesture(frames[ii].gesture_id))
{
cmd.ignore = true;
}
cmds.Add(cmd);
}
// This buffering approach is really
// inefficient, but since this off-line,
// performance is not important.
buffer.Add(pt);
if (buffer.Count > parameters.sliding_window_frame_cnt)
{
buffer.RemoveAt(0);
}
// We need to have a couple points before
// calling the recognizer.
if (buffer.Count < 2)
{
continue;
}
// Wait a few frames again before trying
// to recognize again.
if (frame_no < next_update)
{
continue;
}
next_update = frame_no + parameters.update_interval;
// Run the recognizer.
int gesture_id = jk.Classify(buffer);
// Add recognition result.
detections.Add(gesture_id);
if (detections.Count > parameters.repeat_cnt)
{
detections.RemoveAt(0);
}
// Count how many times this gesture was recognized.
int winner_cnt = 0;
for (int jj = 0;
jj < detections.Count;
jj++)
{
if (detections[jj] == gesture_id)
{
winner_cnt += 1;
}
}
// Ensure we have enough recognitions.
if (winner_cnt < parameters.repeat_cnt)
{
continue;
}
// If nothing was detected, skip rest.
if (gesture_id == -1)
{
continue;
}
// Hurray! A gesture is recognized!
// Hopefully it's the right one too!!
cmds[cmds.Count - 1].add(gesture_id);
detections.Clear();
buffer.Clear();
}
// Mark bad commands, situations where the participant
// made a mistake or tracking was lost. We know the
// command was bad because the protector asked the
// participant to repeat the gesture, but a new command
// ID is assigned to the sequence.
for (int ii = 1;
ii < cmds.Count;
ii++)
{
if (cmds[ii].expected_id == cmds[ii - 1].expected_id)
{
CommandResults temp = cmds[ii - 1];
temp.ignore = true;
cmds[ii - 1] = temp;
}
}
// Put all results in confusion matrices.
ConfusionMatrices ret = new ConfusionMatrices(ds);
for (int ii = 0; ii < cmds.Count; ii++)
{
if (cmds[ii].ignore)
{
continue;
}
cmds[ii].update_confusion_matrices(ret);
}
return(ret);
}
public static Results EvaluateSessionWindowed(DeviceType device, int subject_id)
{
configuartion_parameters_t parameneters = new configuartion_parameters_t(device);
// Load subject dataset
Dataset ds = Global.load_subject_dataset(device, subject_id);
List <Sample> train_set = Global.GetTrainSet(ds, 1);
// Covert the dataset to format accepted by Jackknife
List <Jackknife.Sample> jk_train_set = JackknifeConnector.GetJKTrainSet(train_set);
// Load subject session
List <Frame> frames = new List <Frame>();
Global.load_session(device, subject_id, frames, ds);
// Load ground truth
List <GestureCommand> cmds = new List <GestureCommand>();
GestureCommand.GetAllCommands(cmds, ds, device, subject_id);
// Train the recognizer
JackknifeBlades blades = new JackknifeBlades();
blades.SetIPDefaults();
blades.ResampleCnt = 20;
Jackknife.Jackknife jk = new Jackknife.Jackknife(blades);
foreach (Jackknife.Sample s in jk_train_set)
{
jk.AddTemplate(s);
}
// Set between 2.0 and 10.0 in steps of .25
// to find the best result
jk.SetRejectionThresholds(5.25f);
// Set up filter for session points
ExponentialMovingAverage ema_filter = new ExponentialMovingAverage(frames[0].pt);
Vector pt;
WindowSegmentor windowSegmentor = new WindowSegmentor(jk);
//List<RecognitionResult> rresults = new List<RecognitionResult>();
List <ContinuousResult> continuous_results = new List <ContinuousResult>();
// Go through session
for (int session_pt = 0; session_pt < frames.Count; session_pt++)
{
long ts1 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // at beginning
pt = ema_filter.Filter(frames[session_pt].pt, 1 / (double)parameneters.fps);
long ts2 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after filter
Jackknife.Vector jkpt = JackknifeConnector.ToJKVector(pt);
windowSegmentor.Update(jkpt);
windowSegmentor.Segment(continuous_results);
if (session_pt % 2000 == 0)
{
Debug.Log(string.Format("{0}% Done", (double)session_pt / (double)frames.Count * 100.0));
}
}
foreach (ContinuousResult cr in continuous_results)
{
Debug.Log(string.Format("st {0}, en {1}, gid {2}", cr.startFrameNo, cr.endFrameNo, cr.gid));
}
// Per gesture confusion matrix
List <ConfisionMatrix> cm = new List <ConfisionMatrix>();
for (int ii = 0; ii < ds.Gestures.Count; ii++)
{
cm.Add(new ConfisionMatrix());
}
for (int ii = 0; ii < continuous_results.Count; ii++)
{
ContinuousResult result = continuous_results[ii];
bool found = false;
int cidx = 0;
for (cidx = 0; cidx < cmds.Count; cidx++)
{
found = cmds[cidx].Hit(result);
if (found == true)
{
break;
}
}
if (found == true)
{
// true positive
if (cmds[cidx].detected == false)
{
cmds[cidx].detected = true;
cm[result.gid].tp += 1.0f;
}
}
else
{
bool bad = GestureCommand.IsBadCommand(
frames,
result);
if (bad == true)
{
continue;
}
// false positive
cm[result.gid].fp += 1.0f;
}
}
// false negatives
for (int cidx = 0; cidx < cmds.Count; cidx++)
{
if (cmds[cidx].detected == true)
{
continue;
}
cm[cmds[cidx].gid].fn += 1.0;
}
Results ret = new Results();
for (int ii = 0; ii < cm.Count; ii++)
{
ret.AppendResults(cm[ii]);
}
ret.PrintF();
return(ret);
}
public static Results EvaluateSession(DeviceType device, int subject_id)
{
configuartion_parameters_t parameneters = new configuartion_parameters_t(device);
// Load subject dataset
Dataset ds = Global.load_subject_dataset(device, subject_id);
List <Sample> train_set = Global.GetTrainSet(ds, 1);
// Covert the dataset to format accepted by Jackknife
List <Jackknife.Sample> jk_train_set = JackknifeConnector.GetJKTrainSet(train_set);
// Load subject session
List <Frame> frames = new List <Frame>();
Global.load_session(device, subject_id, frames, ds);
//Debug.Log("Frame_cnt = " + frames.Count());
// Load ground truth
List <GestureCommand> cmds = new List <GestureCommand>();
GestureCommand.GetAllCommands(cmds, ds, device, subject_id);
// Train the segmentor
ContinuousResultOptions cr_options = new ContinuousResultOptions();
//COREY FIX latency framecount
cr_options.latencyFrameCount = 1;
Machete yeah = new Machete(device, cr_options);
foreach (Sample s in train_set)
{
yeah.AddSample(s, filtered: true);
}
//PrintYeahStats(yeah);
// Train the recognizer
JackknifeBlades blades = new JackknifeBlades();
blades.SetIPDefaults();
//COREY FIX RESAMPLECNT
blades.ResampleCnt = 20;
blades.LowerBound = false;
Jackknife.Jackknife jk = new Jackknife.Jackknife(blades);
foreach (Jackknife.Sample s in jk_train_set)
{
jk.AddTemplate(s);
}
// Set between 2.0 and 10.0 in steps of .25
// to find the best result
// Best at 7.5 with around 66% :/
jk.SetRejectionThresholds(7.0);
// Set up filter for session points
ExponentialMovingAverage ema_filter = new ExponentialMovingAverage(frames[0].pt, 5.0);
Vector pt;
List <Vector> video = new List <Vector>();
List <RecognitionResult> rresults = new List <RecognitionResult>();
int triggered_count = 0;
// Go through session
for (int session_pt = 0; session_pt < frames.Count; session_pt++)
{
long ts1 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // at beginning
List <ContinuousResult> continuous_results = new List <ContinuousResult>();
pt = ema_filter.Filter(frames[session_pt].pt, 1 / (double)parameneters.fps);
long ts2 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after filter
video.Add(pt);
//Debug.Log(string.Format("Pt: {0} {1} {2}", pt.Data[0], pt.Data[1], pt.Data[2]));
yeah.ProcessFrame(pt, session_pt, continuous_results);
long ts3 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after processing frame
bool cancel_if_better_score = false;
ContinuousResult result = ContinuousResult.SelectResult(
continuous_results,
cancel_if_better_score);
long ts4 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after looking for result
//Debug.Log(string.Format("{0} {1} {2}", ts2 - ts1, ts3 - ts2, ts4 - ts3));
//Debug.Log(string.Format("FRAME NO: {0}", frame_no));
if (result == null)
{
continue;
}
//COREY FIX For comparing against the original code
//if (result.sample.GestureId == 0)
// Debug.Log(string.Format("start {0}, end {1}", result.startFrameNo, result.endFrameNo + 1));
triggered_count += 1;
//Debug.Log(string.Format("Frame: {3} Result: {0}, Sample: {1}, Score: {2}", result.gid, result.sample.GestureName, result.score, frame_no));
//Debug.Log(string.Format("Best result as of: {0} {1}", ii, yeah.bestScore));
// Run recognizer on segmented result
double recognizer_d = 0.0f;
bool match = false;
// Save a buffer to pass to recognizer
List <Jackknife.Vector> jkbuffer = JackknifeConnector.GetJKBufferFromVideo(
video,
result.startFrameNo,
result.endFrameNo);
long ts5 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // before passing to recognizer
match = jk.IsMatch(jkbuffer, result.sample.GestureId, out recognizer_d);
//COREY Fix print out scores
if (result.sample.GestureId == 0)
{
//Debug.Log(string.Format("start {0}, end {1} ", result.startFrameNo, result.endFrameNo + 1) + string.Format("Is match = {0}, score = {1}",
// match,
// recognizer_d));
}
// Matched to template with this gid
if (match == false)
{
continue;
}
long ts6 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after classifying
// Gesture was accepted
RecognitionResult rresult = new RecognitionResult();
rresult.gid = result.gid;
rresult.start = result.startFrameNo;
rresult.end = result.endFrameNo;
rresult.score = recognizer_d;
match = false;
for (int ii = 0; ii < rresults.Count; ii++)
{
if (rresults[ii].Update(rresult) == true)
{
match = true;
break;
}
}
// if some result was updated for better, continue
if (match == true)
{
continue;
}
rresults.Add(rresult);
}
// Per gesture confusion matrix
List <ConfisionMatrix> cm = new List <ConfisionMatrix>();
for (int ii = 0; ii < ds.Gestures.Count; ii++)
{
cm.Add(new ConfisionMatrix());
}
for (int ii = 0; ii < rresults.Count; ii++)
{
RecognitionResult result = rresults[ii];
bool found = false;
int cidx = 0;
for (cidx = 0; cidx < cmds.Count; cidx++)
{
found = cmds[cidx].Hit(result);
if (found == true)
{
break;
}
}
if (found == true)
{
// true positive
if (cmds[cidx].detected == false)
{
cmds[cidx].detected = true;
cm[result.gid].tp += 1.0f;
}
}
else
{
bool bad = GestureCommand.IsBadCommand(
frames,
result);
if (bad == true)
{
continue;
}
// false positive
cm[result.gid].fp += 1.0f;
}
}
// false negatives
for (int cidx = 0; cidx < cmds.Count; cidx++)
{
if (cmds[cidx].detected == true)
{
continue;
}
cm[cmds[cidx].gid].fn += 1.0;
}
Results ret = new Results();
for (int ii = 0; ii < cm.Count; ii++)
{
ret.AppendResults(cm[ii]);
//temp += string.Format("{5}:\t A: {0:N6}, E: {1:N6}, P: {2:N6}, R: {3:N6}, F1: {4:N6}\n", ret.accuracy/ret.total, ret.error / ret.total, ret.precision / ret.total, ret.recall / ret.total, ret.f1_0 / ret.total, ii);
}
//Debug.Log(temp);
return(ret);
}
/**
* A simple user independent test.
*/
public static void user_indepedent_test(DeviceType device)
{
// First, load all training data for one device type.
string path = "";
if (device == DeviceType.KINECT)
{
path = Global.GetRootDirectory() + "datasets/jk2017/kinect/training/";
}
else if (device == DeviceType.LEAP_MOTION)
{
path = Global.GetRootDirectory() + "datasets/jk2017/leap_motion/training/";
}
Dataset ds = Dataset.LoadDataset(path);
int subject_cnt = ds.Subjects.Count;
int sample_cnt = ds.Samples.Count;
ConfusionMatrices cm = new ConfusionMatrices(ds);
// iterate through all subjects
for (int subject_id = 0; subject_id < subject_cnt; subject_id++)
{
ConfusionMatrices cm_individual = new ConfusionMatrices(ds);
Console.WriteLine("Participant: " + subject_id);
// train a recognizer with the selected subject
JackknifeBlades blades = new JackknifeBlades();
blades.SetIPDefaults();
Jackknife jk = new Jackknife(blades);
for (int sample_id = 0; sample_id < sample_cnt; sample_id++)
{
Sample sample = ds.Samples[sample_id];
if (sample.SubjectId != subject_id)
{
continue;
}
jk.AddTemplate(ds.Samples[sample_id]);
}
// only train the recognize if you need
// test the recognizer with all other samples
for (int sample_id = 0; sample_id < sample_cnt; sample_id++)
{
Sample sample = ds.Samples[sample_id];
if (sample.SubjectId == subject_id)
{
continue;
}
int gid = jk.Classify(sample);
cm_individual.AddResult(
sample.GestureId,
gid);
}
cm.AddResult(cm_individual);
ResultT resultss = cm_individual.Results();
resultss.Print();
}
Console.WriteLine("Aggregate Results: ");
ResultT results = cm.Results();
results.Print();
}