public OnBodyInputRealtimeExperiments()
{
InitializeComponent();
Location = Properties.Settings.Default.MainLocation;
Size = Properties.Settings.Default.MainSize;
captureSound = new SoundPlayer("sounds\\camera_capture.wav");
tickSound = new SoundPlayer("sounds\\tick.wav");
beepSound = new SoundPlayer("sounds\\camera_beep.wav");
phoneSound = new SoundPlayer("sounds\\phone.wav");
speech.Rate = 2;
captureSound.LoadAsync();
tickSound.LoadAsync();
beepSound.LoadAsync();
timer.Elapsed += Timer_Elapsed;
CountdownLabel.Parent = Display;
numLocationPredictions = Properties.Settings.Default.PredictionSmoothing;
Text = "Connecting to Camera and Sensors...";
// set up the gestures and menus
GestureActionMap.LoadMacros(File.ReadAllText("defaults/macros.txt"));
GestureActionMap.LoadMenus(File.ReadAllText("defaults/menus.txt"));
GestureActionMap.LoadActions(File.ReadAllText("defaults/actions.txt"));
// needs to be initialized after the GestureActionMap
testingForm = new TestingForm();
testingForm.TaskStarted += (string task) =>
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(TestingVoice);
speech.SpeakAsync("Begin");
};
testingForm.TaskFinished += (string task) =>
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(TestingVoice);
speech.SpeakAsync("Task Completed");
};
//trainingForm.TrainingDataUpdated += () => { UpdateTrainingLabel(); };
trainingForm.RecordLocation += (string coarseLocation, string fineLocation) =>
{
Logging.LogTrainingEvent("prepare_to_capture_location");
coarseLocationToTrain = coarseLocation;
fineLocationToTrain = fineLocation;
// change the countdown timer setting if you don't want to capture the sample immediately (useful if you're training on yourself, for example)
countdown = Properties.Settings.Default.CountdownTimer;
if (countdown > 0)
{
tickSound.Play();
CountdownLabel.Text = countdown.ToString();
CountdownLabel.Visible = true;
timer.Start();
}
else
{
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play();
}
ImageTemplate newTemplate = AddTemplate();
Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]);
//trainingForm.UpdateLocationList();
}
};
trainingForm.RecordGesture += (string gesture) =>
{
Logging.LogTrainingEvent("start_recording_gesture");
if (Properties.Settings.Default.EnableSoundEffects)
{
beepSound.Play();
}
gestureToTrain = gesture;
recordingGesture = true;
};
trainingForm.AutoCaptureGesture += (string gesture) =>
{
Logging.LogTrainingEvent("start_autocapture_gesture");
if (Properties.Settings.Default.EnableSoundEffects)
{
beepSound.Play();
}
gestureToTrain = gesture;
autoTrainGesture = true;
};
trainingForm.StopAutoCapturingGestures += () =>
{
Logging.LogTrainingEvent("stop_autocapture_gesture");
autoTrainGesture = false;
GestureRecognition.Train();
trainingForm.UpdateGestureList();
};
trainingForm.AutoCaptureLocation += (string coarseLocation, string fineLocation) =>
{
coarseLocationToTrain = coarseLocation;
fineLocationToTrain = fineLocation;
Logging.LogTrainingEvent("start_autocapture_location");
countdown = Properties.Settings.Default.CountdownTimer;
if (countdown > 0)
{
prepareToAutoTrainLocation = true;
tickSound.Play();
CountdownLabel.Text = countdown.ToString();
CountdownLabel.Visible = true;
timer.Start();
}
else
{
if (Properties.Settings.Default.EnableSoundEffects)
{
beepSound.Play();
}
autoTrainLocation = true;
}
};
trainingForm.StopAutoCapturingLocation += () =>
{
autoTrainLocation = false;
Logging.LogTrainingEvent("stop_autocapture_location");
};
TouchSegmentation.TouchDownEvent += () =>
{
touchDown = true;
hovering = false;
hoverCoarseLocation = null;
hoverFineLocation = null;
touchStart = DateTime.Now;
Sensors.Instance.Brightness = ledMaxBrightness;
Logging.LogOtherEvent("touch_down");
// reset the location predictions
if (!recentTouchUp)
{
while (coarseLocationProbabilities.Count > 0)
{
coarseLocationProbabilities.Take();
}
while (fineLocationProbabilities.Count > 0)
{
fineLocationProbabilities.Take();
}
while (gestureCoarseLocationProbabilities.Count > 0)
{
gestureCoarseLocationProbabilities.Take();
}
while (gestureFineLocationProbabilities.Count > 0)
{
gestureFineLocationProbabilities.Take();
}
while (coarseLocationPredictions.Count > 0)
{
coarseLocationPredictions.Take();
}
while (gestureCoarseLocationPredictions.Count > 0)
{
gestureCoarseLocationPredictions.Take();
}
while (gestureFocusWeights.Count > 0)
{
gestureFocusWeights.Take();
}
while (gestureSensorReadings.Count > 0)
{
gestureSensorReadings.Take();
}
foreach (Sensors.Reading reading in sensorReadingHistory)
{
gestureSensorReadings.Add(reading);
}
Logging.LogOtherEvent("smoothing_reset");
}
Invoke(new MethodInvoker(delegate { TouchStatusLabel.Text = "Touch Down"; }));
};
TouchSegmentation.TouchUpEvent += () =>
{
touchDown = false;
recentTouchUp = true;
Invoke(new MethodInvoker(delegate { TouchStatusLabel.Text = "Touch Up"; }));
Logging.LogOtherEvent("touch_up");
// trigger an event after a short delay to prevent false positives and allow for the double-tap gesture
// it is cancelled if the user touches down again within that time
Task.Factory.StartNew(() =>
{
Thread.Sleep(touchUpDelay);
recentTouchUp = false;
if (!touchDown)
{
Logging.LogOtherEvent("touch_up_timeout");
Sensors.Instance.Brightness = 0;
if (autoTrainLocation)
{
autoTrainLocation = false;
trainingForm.StopAutoCapture();
}
if ((hovering && !recordingGesture && !autoTrainGesture) || trainingForm.Training)
{
return;
}
// process the gesture
Gesture gesture = new Gesture(gestureSensorReadings.ToArray());
if (gestureSensorReadings.Count > 60)
{
GestureRecognition.PreprocessGesture(gesture);
if (recordingGesture || autoTrainGesture)
{
//Monitor.Enter(recognitionLock);
recordingGesture = false;
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play(); Logging.LogAudioEvent("capture", false);
}
DateTime start = DateTime.Now;
gesture.ClassName = gestureToTrain;
GestureRecognition.AddTrainingExample(gesture, gestureToTrain);
Logging.LogTrainingEvent("Add gesture: " + gestureToTrain);
if (!autoTrainGesture)
{
GestureRecognition.Train();
Debug.WriteLine("Training: " + (DateTime.Now - start).TotalMilliseconds + " ms");
}
start = DateTime.Now;
if (!autoTrainGesture)
{
//trainingForm.UpdateGestureList();
trainingForm.AddGesture(gesture);
Debug.WriteLine("Updating List: " + (DateTime.Now - start).TotalMilliseconds + " ms");
}
//Monitor.Exit(recognitionLock);
}
else
{
gesture.ClassName = GestureRecognition.PredictGesture(gesture);
}
}
Monitor.Enter(recognitionLock);
// predict the most likely location over the coarse of the gesture, weighted by voting and image focus
Dictionary <string, float>[] tempCoarseProbabilities = gestureCoarseLocationProbabilities.ToArray();
Dictionary <string, float>[] tempFineProbabilities = gestureFineLocationProbabilities.ToArray();
string[] tempCoarsePredictions = gestureCoarseLocationPredictions.ToArray();
float[] tempFocusWeights = gestureFocusWeights.ToArray();
// sum up the probabilities
Dictionary <string, float> totalProbabilities = new Dictionary <string, float>();
//foreach (Dictionary<string, float> probabilities in gestureCoarseLocationProbabilities)
for (int i = 0; i < tempCoarseProbabilities.Length; i++)
{
Dictionary <string, float> probabilities = tempCoarseProbabilities[i];
float weight = Math.Max(tempFocusWeights.Length > i ? tempFocusWeights[i] : 0.01f, 0.01f);
foreach (string key in probabilities.Keys)
{
if (!totalProbabilities.ContainsKey(key))
{
totalProbabilities[key] = 0;
}
totalProbabilities[key] += weight * probabilities[key] / numLocationPredictions;
}
}
float maxProb = 0;
string coarseLocation = "";
float coarseProbability = 0;
foreach (string key in totalProbabilities.Keys)
{
if (totalProbabilities[key] > maxProb)
{
maxProb = totalProbabilities[key];
coarseLocation = key;
coarseProbability = maxProb;
}
}
// sum up the probabilities
totalProbabilities = new Dictionary <string, float>();
//foreach (Dictionary<string, float> probabilities in gestureFineLocationProbabilities)
for (int i = 0; i < tempFineProbabilities.Length; i++)
{
if (i < tempCoarsePredictions.Length && tempCoarsePredictions[i] == coarseLocation)
{
Dictionary <string, float> probabilities = tempFineProbabilities[i];
float weight = Math.Max(tempFocusWeights.Length > i ? tempFocusWeights[i] : 0.01f, 0.01f);
foreach (string key in probabilities.Keys)
{
if (!totalProbabilities.ContainsKey(key))
{
totalProbabilities[key] = 0;
}
totalProbabilities[key] += weight * probabilities[key] / numLocationPredictions;
}
}
}
maxProb = 0;
string fineLocation = "";
float fineProbability = 0;
foreach (string key in totalProbabilities.Keys)
{
if (totalProbabilities[key] > maxProb)
{
maxProb = totalProbabilities[key];
fineLocation = key;
fineProbability = maxProb;
}
}
Logging.LogGestureEvent(gesture.ClassName, coarseLocation + " " + fineLocation, gesture);
Monitor.Exit(recognitionLock);
if (fineLocation == "")
{
Debug.WriteLine("Error");
}
Invoke(new MethodInvoker(delegate
{
CoarsePredictionLabel.Text = "= " + coarseLocation;
CoarseProbabilityLabel.Text = " (response = " + (coarseProbability * 100).ToString("0.0") + ")";
FinePredictionLabel.Text = "= " + fineLocation;
FineProbabilityLabel.Text = " (response = " + (fineProbability * 100).ToString("0.0") + ")";
GesturePredictionLabel.Text = gesture.ClassName;
if (Properties.Settings.Default.EnableSpeechOutput)
{
if (Properties.Settings.Default.EnableApplicationDemos)
{
string actionResult = GestureActionMap.PerformAction(gesture.ClassName, coarseLocation, fineLocation, Properties.Settings.Default.GestureMode, Properties.Settings.Default.FixedApplicationResponses, lockToCurrentTask: Properties.Settings.Default.LockToCurrentTask);
if (actionResult != null && actionResult.Length > 0)
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(MenuVoice);
//speech.SpeakAsync(gesture.ClassName + " " + coarseLocation + " " + fineLocation);
speech.SpeakAsync(actionResult);
Logging.LogAudioEvent(actionResult);
}
}
else
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(MenuVoice);
speech.SpeakAsync(gesture.ClassName + " " + coarseLocation + " " + fineLocation);
Logging.LogAudioEvent(gesture.ClassName + " " + coarseLocation + " " + fineLocation);
}
}
}));
}
});
};
// Initialize the camera and other sensors in a background thread to avoid hanging the UI
Task.Factory.StartNew(() =>
{
Camera.Instance.FrameAvailable += Camera_FrameAvailable;
Camera.Instance.Error += (string error) =>
{
Debug.WriteLine(error);
Invoke(new MethodInvoker(delegate { Text = "Error: could not connect to camera"; }));
};
Camera.Instance.Brightness = Properties.Settings.Default.CameraBrightness;
Sensors.Instance.ReadingAvailable += Sensors_ReadingAvailable;
Camera.Instance.Connect(); Logging.LogOtherEvent("camera_connected");
Sensors.Instance.Connect(Properties.Settings.Default.SensorPort); Logging.LogOtherEvent("sensors_connected");
Sensors.Instance.NumSensors = Properties.Settings.Default.SingleIMU ? 1 : 2;
Invoke(new MethodInvoker(delegate
{
if (Properties.Settings.Default.TrainingVisible)
{
trainingToolStripMenuItem.PerformClick();
}
if (Properties.Settings.Default.SettingsVisible)
{
settingsToolStripMenuItem.PerformClick();
}
if (Properties.Settings.Default.TestingVisible)
{
testingToolStripMenuItem.PerformClick();
}
}));
});
}
public LocationAndGestureDataCollectionTool()
{
InitializeComponent();
Location = Properties.Settings.Default.MainLocation;
Size = Properties.Settings.Default.MainSize;
captureSound = new SoundPlayer("sounds\\camera_capture.wav");
beepSound = new SoundPlayer("sounds\\camera_beep.wav");
chimeSound = new SoundPlayer("sounds\\endTask.wav");
speech.Rate = 2;
captureSound.LoadAsync();
beepSound.LoadAsync();
chimeSound.LoadAsync();
Text = "Connecting to Camera and Sensors...";
trainingForm.RecordLocation += (string coarseLocation, string fineLocation) =>
{
Logging.LogTrainingEvent("prepare_to_capture_location");
coarseLocationToTrain = coarseLocation;
fineLocationToTrain = fineLocation;
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play();
}
ImageTemplate newTemplate = AddTemplate();
Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]);
};
trainingForm.RecordGesture += (string gesture) =>
{
Logging.LogTrainingEvent("start_recording_gesture");
if (Properties.Settings.Default.EnableSoundEffects)
{
beepSound.Play();
}
gestureToTrain = gesture;
recordingGesture = true;
};
trainingForm.AutoCaptureGesture += (string gesture) =>
{
Logging.LogTrainingEvent("start_autocapture_gesture");
if (Properties.Settings.Default.EnableSoundEffects)
{
beepSound.Play();
}
gestureToTrain = gesture;
numAutoSamplesGathered = 0;
autoTrainGesture = true;
};
trainingForm.StopAutoCapturingGestures += () =>
{
Logging.LogTrainingEvent("stop_autocapture_gesture");
autoTrainGesture = false;
GestureRecognition.Train();
trainingForm.UpdateGestureList();
};
trainingForm.AutoCaptureLocation += (string coarseLocation, string fineLocation) =>
{
coarseLocationToTrain = coarseLocation;
fineLocationToTrain = fineLocation;
Logging.LogTrainingEvent("start_autocapture_location");
if (Properties.Settings.Default.EnableSoundEffects)
{
beepSound.Play();
}
autoTrainLocation = true;
};
trainingForm.StopAutoCapturingLocation += () =>
{
autoTrainLocation = false;
Logging.LogTrainingEvent("stop_autocapture_location");
};
TouchSegmentation.TouchDownEvent += () =>
{
touchDown = true;
touchStart = DateTime.Now;
Sensors.Instance.Brightness = 1;
Logging.LogTouchEvent("touch_down");
// reset the location predictions
if (!recentTouchUp)
{
while (gestureSensorReadings.Count > 0)
{
gestureSensorReadings.Take();
}
foreach (Sensors.Reading reading in sensorReadingHistory)
{
gestureSensorReadings.Add(reading);
}
Logging.LogOtherEvent("smoothing_reset");
}
Invoke(new MethodInvoker(delegate { TouchStatusLabel.Text = "Touch Down"; }));
};
TouchSegmentation.TouchUpEvent += () =>
{
touchDown = false;
recentTouchUp = true;
Invoke(new MethodInvoker(delegate { TouchStatusLabel.Text = "Touch Up"; }));
Logging.LogTouchEvent("touch_up");
Task.Factory.StartNew(() =>
{
Thread.Sleep(touchUpDelay);
recentTouchUp = false;
if (!touchDown)
{
Logging.LogOtherEvent("touch_up_timeout");
Sensors.Instance.Brightness = 0;
if (autoTrainLocation)
{
autoTrainLocation = false;
trainingForm.StopAutoCapture();
}
if (trainingForm.Training)
{
return;
}
// process the gesture
Gesture gesture = new Gesture(gestureSensorReadings.ToArray());
if (gestureSensorReadings.Count > 60)
{
GestureRecognition.PreprocessGesture(gesture);
if (recordingGesture || autoTrainGesture)
{
//Monitor.Enter(recognitionLock);
if (recordingGesture)
{
Logging.LogTrainingEvent("stop_recording_gesture");
}
recordingGesture = false;
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play(); Logging.LogAudioEvent("capture", false);
}
DateTime start = DateTime.Now;
gesture.ClassName = gestureToTrain;
gesture.IsTrainingData = true;
GestureRecognition.AddTrainingExample(gesture, gestureToTrain);
// autosave the gesture in the background
Task.Factory.StartNew(() =>
{
if (!Directory.Exists(Path.Combine("savedProfiles", "autosave")))
{
Directory.CreateDirectory(Path.Combine("savedProfiles", "autosave"));
}
int index = 1;
while (File.Exists(Path.Combine("savedProfiles", "autosave", gestureToTrain + "_" + index + ".png")))
{
index++;
}
gesture.Path = Path.Combine("savedProfiles", "autosave", gestureToTrain + "_" + index + ".png");
string json = JsonConvert.SerializeObject(gesture);
Invoke(new MethodInvoker(delegate { File.WriteAllText(gesture.Path, json); }));
});
Logging.LogTrainingEvent("Add gesture: " + gestureToTrain + ", path = " + gesture.Path);
if (autoTrainGesture)
{
numAutoSamplesGathered++;
if (numAutoSamplesGathered >= Properties.Settings.Default.NumAutoGestureSamples)
{
if (Properties.Settings.Default.EnableSoundEffects)
{
chimeSound.Play(); Logging.LogAudioEvent("chime", false);
}
Logging.LogTrainingEvent("stop_autocapture_gesture");
autoTrainGesture = false;
}
}
start = DateTime.Now;
if (!autoTrainGesture)
{
trainingForm.AddGesture(gesture);
trainingForm.IsSaved = false;
Debug.WriteLine("Updating List: " + (DateTime.Now - start).TotalMilliseconds + " ms");
}
}
}
}
});
};
Task.Factory.StartNew(() =>
{
Camera.Instance.FrameAvailable += Camera_FrameAvailable;
Camera.Instance.Error += (string error) =>
{
Debug.WriteLine(error);
Invoke(new MethodInvoker(delegate { Text = "Error: could not connect to camera"; }));
};
Camera.Instance.Brightness = Properties.Settings.Default.CameraBrightness;
Sensors.Instance.ReadingAvailable += Sensors_ReadingAvailable;
Camera.Instance.Connect(); Logging.LogOtherEvent("camera_connected");
Sensors.Instance.Connect(Properties.Settings.Default.SensorPort); Logging.LogOtherEvent("sensors_connected");
Sensors.Instance.NumSensors = Properties.Settings.Default.SingleIMU ? 1 : 2;
Invoke(new MethodInvoker(delegate
{
if (Properties.Settings.Default.TrainingVisible)
{
trainingToolStripMenuItem.PerformClick();
}
if (Properties.Settings.Default.SettingsVisible)
{
settingsToolStripMenuItem.PerformClick();
}
}));
});
}
private void RemoveButton_Click(object sender, EventArgs e)
{
if (TrainingDataViewer.SelectedIndex == 0) // location tab is showing
{
foreach (ListViewItem removeItem in LocationView.SelectedItems)
{
int imageIndex = removeItem.ImageIndex;
ImageTemplate template = (ImageTemplate)removeItem.Tag;
Localization.Instance.RemoveTrainingExample(template);
Logging.LogOtherEvent("Removed location example: " + (string)template["path"]);
LocationView.LargeImageList.Images.RemoveAt(imageIndex);
foreach (ListViewItem updateItem in LocationView.Items)
{
if (updateItem.ImageIndex > imageIndex)
{
updateItem.ImageIndex--;
}
}
if (removeItem.Group.Items.Count == 0)
{
LocationView.Groups.Remove(removeItem.Group);
}
LocationView.Items.Remove(removeItem);
}
training = true;
Localization.Instance.Train();
UpdateLocationCount();
training = false;
OnTrainingDataUpdated();
}
else // gesture tab is showing
{
foreach (ListViewItem removeItem in GestureView.SelectedItems)
{
int imageIndex = removeItem.ImageIndex;
Gesture template = (Gesture)removeItem.Tag;
Logging.LogOtherEvent("Removed gesture example: " + template.Path);
GestureRecognition.RemoveTrainingExample(template);
GestureView.LargeImageList.Images.RemoveAt(imageIndex);
foreach (ListViewItem updateItem in GestureView.Items)
{
if (updateItem.ImageIndex > imageIndex)
{
updateItem.ImageIndex--;
}
}
if (removeItem.Group.Items.Count == 0)
{
GestureView.Groups.Remove(removeItem.Group);
}
GestureView.Items.Remove(removeItem);
}
training = true;
GestureRecognition.Train();
UpdateGestureCount();
training = false;
OnTrainingDataUpdated();
}
}
