void Camera_FrameAvailable(VideoFrame frame)
{
if (closing)
{
return;
}
ImageTemplate template = new ImageTemplate(frame.Clone());
FPS.Camera.Update();
Task.Factory.StartNew(() =>
{
try
{
Logging.LogVideoFrame(frame.Image.Bitmap);
float focus = 0;
if (Monitor.TryEnter(preprocessingLock))
{
ImageProcessing.ProcessTemplate(template, false);
focus = ImageProcessing.ImageFocus(template);
Logging.LogFrameProcessed();
currTemplate = template;
FPS.Instance("processing").Update();
Monitor.Exit(preprocessingLock);
}
else
{
return;
}
string coarseLocation = "", fineLocation = "";
float coarseProbability = 0, fineProbability = 0;
bool hasUpdate = false;
string predictedCoarseLocation = "", predictedFineLocation = "";
if (touchDown && !trainingForm.Training && Monitor.TryEnter(recognitionLock))
{
try
{
if (Localization.Instance.GetNumTrainingExamples() > 0)
{
Dictionary <string, float> coarseProbabilities = new Dictionary <string, float>();
predictedCoarseLocation = Localization.Instance.PredictCoarseLocation(currTemplate, out coarseProbabilities);
coarseLocation = predictedCoarseLocation;
if (!Properties.Settings.Default.CoarseOnly)
{
bool foundFeatureMatch = false;
Dictionary <string, float> fineProbabilities = new Dictionary <string, float>();
predictedFineLocation = Localization.Instance.PredictFineLocation(currTemplate, out foundFeatureMatch, out fineProbabilities, true, false, false, coarseLocation);
fineLocation = predictedFineLocation;
}
Logging.LogLocationEvent(predictedCoarseLocation + " " + predictedFineLocation);
FPS.Instance("matching").Update();
hasUpdate = true;
}
if (autoTrainLocation && (coarseLocation != coarseLocationToTrain || fineLocation != fineLocationToTrain) && Monitor.TryEnter(trainingLock))
{
trainingForm.Training = true;
ImageTemplate newTemplate = CopyTemplate(currTemplate);
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = true; }));
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play(); Logging.LogAudioEvent("capture", false);
}
AddTemplate(newTemplate);
Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]);
Thread.Sleep(100);
//if (Properties.Settings.Default.EnableSoundEffects) { beepSound.Play(); Logging.LogAudioEvent("beep", false); }
trainingForm.IsSaved = false;
trainingForm.Training = false;
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = false; }));
Monitor.Exit(trainingLock);
return;
}
}
finally
{
Monitor.Exit(recognitionLock);
}
}
//LBP.GetInstance(frame.Image.Size).GetHistogram(frame);
Invoke(new MethodInvoker(delegate
{
Display.Image = frame.Image.Bitmap;
if (hasUpdate)
{
CoarsePredictionLabel.Text = coarseLocation;
FinePredictionLabel.Text = fineLocation;
}
Text = FPS.Camera.Average.ToString("0") + " fps camera / " + (Sensors.Instance.IsConnected ? FPS.Sensors.Average.ToString("0") + " fps sensors / " + FPS.Instance("processing").Average.ToString("0") + "/" + FPS.Instance("matching").Average.ToString("0") + " fps processing" : "Waiting for Sensors") /*+ " / focus = " + focus.ToString("0")*/;
}));
}
catch { }
});
}
void Camera_FrameAvailable(VideoFrame frame)
{
if (closing)
{
return;
}
numLocationPredictions = Properties.Settings.Default.TestingMode.Contains("Training") ? 1 : Properties.Settings.Default.PredictionSmoothing;
ImageTemplate template = new ImageTemplate(frame.Clone());
FPS.Camera.Update();
Task.Factory.StartNew(() =>
{
try
{
Logging.LogVideoFrame(frame.Image.Bitmap);
float focus = 0;
//lock (processingLock)
if (Monitor.TryEnter(preprocessingLock))
{
ImageProcessing.ProcessTemplate(template, false);
focus = ImageProcessing.ImageFocus(template);
Logging.LogFrameProcessed();
currTemplate = template;
FPS.Instance("processing").Update();
Monitor.Exit(preprocessingLock);
}
else
{
return;
}
string coarseLocation = "", fineLocation = "";
float coarseProbability = 0, fineProbability = 0;
bool hasUpdate = false;
string predictedCoarseLocation = "", predictedFineLocation = "";
if (touchDown && !trainingForm.Training && Monitor.TryEnter(recognitionLock))
{
try
{
if (Localization.Instance.GetNumTrainingExamples() > 0)
{
Dictionary <string, float> coarseProbabilities = new Dictionary <string, float>();
predictedCoarseLocation = Localization.Instance.PredictCoarseLocation(currTemplate, out coarseProbabilities);
//coarseLocationPredictions.Add(predictedCoarseLocation);
//while (coarseLocationPredictions.Count > numLocationPredictions) coarseLocationPredictions.Take();
coarseLocationProbabilities.Add(coarseProbabilities);
gestureCoarseLocationProbabilities.Add(coarseProbabilities);
while (coarseLocationProbabilities.Count > numLocationPredictions)
{
coarseLocationProbabilities.Take();
}
//// compute the mode of the array
//var groups = coarseLocationPredictions.GroupBy(v => v);
//int maxCount = groups.Max(g => g.Count());
//coarseLocation = groups.First(g => g.Count() == maxCount).Key;
// sum up the probabilities
Dictionary <string, float> totalProbabilities = new Dictionary <string, float>();
foreach (Dictionary <string, float> probabilities in coarseLocationProbabilities)
{
foreach (string key in probabilities.Keys)
{
if (!totalProbabilities.ContainsKey(key))
{
totalProbabilities[key] = 0;
}
totalProbabilities[key] += probabilities[key] / numLocationPredictions;
}
}
float maxProb = 0;
foreach (string key in totalProbabilities.Keys)
{
if (totalProbabilities[key] > maxProb)
{
maxProb = totalProbabilities[key];
coarseLocation = key;
coarseProbability = maxProb;
}
}
coarseLocationPredictions.Add(coarseLocation);
gestureCoarseLocationPredictions.Add(coarseLocation);
while (coarseLocationPredictions.Count > numLocationPredictions)
{
coarseLocationPredictions.Take();
}
if (!Properties.Settings.Default.CoarseOnly)
{
bool foundFeatureMatch = false;
Dictionary <string, float> fineProbabilities = new Dictionary <string, float>();
predictedFineLocation = Localization.Instance.PredictFineLocation(currTemplate, out foundFeatureMatch, out fineProbabilities, true, false, false, coarseLocation);
//fineLocationPredictions.Add(predictedFineLocation);
//while (fineLocationPredictions.Count > numLocationPredictions) fineLocationPredictions.Take();
fineLocationProbabilities.Add(fineProbabilities);
gestureFineLocationProbabilities.Add(fineProbabilities);
while (fineLocationProbabilities.Count > numLocationPredictions)
{
fineLocationProbabilities.Take();
}
// compute the mode of the array
//var groups = fineLocationPredictions.GroupBy(v => v);
//int maxCount = groups.Max(g => g.Count());
//fineLocation = groups.First(g => g.Count() == maxCount).Key;
// sum up the probabilities
totalProbabilities = new Dictionary <string, float>();
//foreach (Dictionary<string, float> probabilities in fineLocationProbabilities)
for (int fineIndex = 0; fineIndex < fineLocationProbabilities.Count; fineIndex++)
{
Dictionary <string, float> probabilities = fineLocationProbabilities.ToArray()[fineIndex];
// make sure that the fine prediction matches the classes for the coarse prediction
if (coarseLocationPredictions.ToArray()[fineIndex] == coarseLocation)
{
foreach (string key in probabilities.Keys)
{
if (!totalProbabilities.ContainsKey(key))
{
totalProbabilities[key] = 0;
}
totalProbabilities[key] += probabilities[key] / numLocationPredictions;
}
}
}
maxProb = 0;
foreach (string key in totalProbabilities.Keys)
{
if (totalProbabilities[key] > maxProb)
{
maxProb = totalProbabilities[key];
fineLocation = key;
fineProbability = maxProb;
}
}
}
Logging.LogLocationEvent(predictedCoarseLocation + " " + predictedFineLocation, coarseLocation + " " + fineLocation);
gestureFocusWeights.Add(focus);
FPS.Instance("matching").Update();
hasUpdate = true;
}
if (autoTrainLocation && (coarseLocation != coarseLocationToTrain || fineLocation != fineLocationToTrain) && Monitor.TryEnter(trainingLock))
{
trainingForm.Training = true;
ImageTemplate newTemplate = CopyTemplate(currTemplate);
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = true; }));
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play(); Logging.LogAudioEvent("capture", false);
}
AddTemplate(newTemplate);
Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]);
Thread.Sleep(100);
//if (Properties.Settings.Default.EnableSoundEffects) { beepSound.Play(); Logging.LogAudioEvent("beep", false); }
trainingForm.Training = false;
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = false; }));
Monitor.Exit(trainingLock);
return;
}
if ((DateTime.Now - touchStart).TotalMilliseconds > Properties.Settings.Default.HoverTimeThreshold)
{
hovering = true;
//if (hoverCoarseLocation == null || coarseLocation == hoverCoarseLocation) // make sure we have the same coarse location, to help prevent jumping around
{
hoverCoarseLocation = coarseLocation;
if (hoverFineLocation == null || fineLocation != hoverFineLocation) // make sure we haven't reported the fine location already
{
hoverFineLocation = fineLocation;
Debug.WriteLine(hoverCoarseLocation + " " + hoverFineLocation);
Invoke(new MethodInvoker(delegate { GesturePredictionLabel.Text = "Hover"; }));
if (Properties.Settings.Default.EnableSpeechOutput)
{
if (Properties.Settings.Default.EnableApplicationDemos)
{
string actionResult = GestureActionMap.PerformAction("Hover", coarseLocation, fineLocation, Properties.Settings.Default.GestureMode, Properties.Settings.Default.FixedApplicationResponses);
if (actionResult != null && actionResult.Length > 0)
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(MenuVoice);
speech.SpeakAsync(actionResult);
Logging.LogAudioEvent(actionResult);
}
}
else
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(MenuVoice);
speech.SpeakAsync("Hover " + coarseLocation + " " + fineLocation);
Logging.LogAudioEvent("Hover " + coarseLocation + " " + fineLocation);
}
}
}
}
}
}
finally
{
Monitor.Exit(recognitionLock);
}
}
//LBP.GetInstance(frame.Image.Size).GetHistogram(frame);
Invoke(new MethodInvoker(delegate
{
Display.Image = frame.Image.Bitmap;
if (hasUpdate)
{
CoarsePredictionLabel.Text = coarseLocation;
CoarseProbabilityLabel.Text = " (response = " + (coarseProbability * 100).ToString("0.0") + ")";
//Debug.WriteLine(coarseLocation);
FinePredictionLabel.Text = fineLocation;
FineProbabilityLabel.Text = " (response = " + (fineProbability * 100).ToString("0.0") + ")";
//Debug.WriteLine(fineLocation);
}
//else if(!touchDown)
//{
// CoarsePredictionLabel.Text = "";
// CoarseProbabilityLabel.Text = "";
// FinePredictionLabel.Text = "";
// FineProbabilityLabel.Text = "";
//}
//focus = (int)(focus / 100) * 100;
Text = FPS.Camera.Average.ToString("0") + " fps camera / " + (Sensors.Instance.IsConnected ? FPS.Sensors.Average.ToString("0") + " fps sensors / " + FPS.Instance("processing").Average.ToString("0") + " fps processing / " + FPS.Instance("matching").Average.ToString("0") + " fps matching" : "Waiting for Sensors") /*+ " / focus = " + focus.ToString("0")*/;
}));
}
catch { }
});
}