void Camera_FrameAvailable(VideoFrame frame) { if (closing) { return; } ImageTemplate template = new ImageTemplate(frame.Clone()); FPS.Camera.Update(); Task.Factory.StartNew(() => { try { Logging.LogVideoFrame(frame.Image.Bitmap); float focus = 0; if (Monitor.TryEnter(preprocessingLock)) { ImageProcessing.ProcessTemplate(template, false); focus = ImageProcessing.ImageFocus(template); Logging.LogFrameProcessed(); currTemplate = template; FPS.Instance("processing").Update(); Monitor.Exit(preprocessingLock); } else { return; } string coarseLocation = "", fineLocation = ""; float coarseProbability = 0, fineProbability = 0; bool hasUpdate = false; string predictedCoarseLocation = "", predictedFineLocation = ""; if (touchDown && !trainingForm.Training && Monitor.TryEnter(recognitionLock)) { try { if (Localization.Instance.GetNumTrainingExamples() > 0) { Dictionary <string, float> coarseProbabilities = new Dictionary <string, float>(); predictedCoarseLocation = Localization.Instance.PredictCoarseLocation(currTemplate, out coarseProbabilities); coarseLocation = predictedCoarseLocation; if (!Properties.Settings.Default.CoarseOnly) { bool foundFeatureMatch = false; Dictionary <string, float> fineProbabilities = new Dictionary <string, float>(); predictedFineLocation = Localization.Instance.PredictFineLocation(currTemplate, out foundFeatureMatch, out fineProbabilities, true, false, false, coarseLocation); fineLocation = predictedFineLocation; } Logging.LogLocationEvent(predictedCoarseLocation + " " + predictedFineLocation); FPS.Instance("matching").Update(); hasUpdate = true; } if (autoTrainLocation && (coarseLocation != coarseLocationToTrain || fineLocation != fineLocationToTrain) && Monitor.TryEnter(trainingLock)) { trainingForm.Training = true; ImageTemplate newTemplate = CopyTemplate(currTemplate); Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = true; })); if (Properties.Settings.Default.EnableSoundEffects) { captureSound.Play(); Logging.LogAudioEvent("capture", false); } AddTemplate(newTemplate); Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]); Thread.Sleep(100); //if (Properties.Settings.Default.EnableSoundEffects) { beepSound.Play(); Logging.LogAudioEvent("beep", false); } trainingForm.IsSaved = false; trainingForm.Training = false; Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = false; })); Monitor.Exit(trainingLock); return; } } finally { Monitor.Exit(recognitionLock); } } //LBP.GetInstance(frame.Image.Size).GetHistogram(frame); Invoke(new MethodInvoker(delegate { Display.Image = frame.Image.Bitmap; if (hasUpdate) { CoarsePredictionLabel.Text = coarseLocation; FinePredictionLabel.Text = fineLocation; } Text = FPS.Camera.Average.ToString("0") + " fps camera / " + (Sensors.Instance.IsConnected ? FPS.Sensors.Average.ToString("0") + " fps sensors / " + FPS.Instance("processing").Average.ToString("0") + "/" + FPS.Instance("matching").Average.ToString("0") + " fps processing" : "Waiting for Sensors") /*+ " / focus = " + focus.ToString("0")*/; })); } catch { } }); }
void Camera_FrameAvailable(VideoFrame frame) { if (closing) { return; } numLocationPredictions = Properties.Settings.Default.TestingMode.Contains("Training") ? 1 : Properties.Settings.Default.PredictionSmoothing; ImageTemplate template = new ImageTemplate(frame.Clone()); FPS.Camera.Update(); Task.Factory.StartNew(() => { try { Logging.LogVideoFrame(frame.Image.Bitmap); float focus = 0; //lock (processingLock) if (Monitor.TryEnter(preprocessingLock)) { ImageProcessing.ProcessTemplate(template, false); focus = ImageProcessing.ImageFocus(template); Logging.LogFrameProcessed(); currTemplate = template; FPS.Instance("processing").Update(); Monitor.Exit(preprocessingLock); } else { return; } string coarseLocation = "", fineLocation = ""; float coarseProbability = 0, fineProbability = 0; bool hasUpdate = false; string predictedCoarseLocation = "", predictedFineLocation = ""; if (touchDown && !trainingForm.Training && Monitor.TryEnter(recognitionLock)) { try { if (Localization.Instance.GetNumTrainingExamples() > 0) { Dictionary <string, float> coarseProbabilities = new Dictionary <string, float>(); predictedCoarseLocation = Localization.Instance.PredictCoarseLocation(currTemplate, out coarseProbabilities); //coarseLocationPredictions.Add(predictedCoarseLocation); //while (coarseLocationPredictions.Count > numLocationPredictions) coarseLocationPredictions.Take(); coarseLocationProbabilities.Add(coarseProbabilities); gestureCoarseLocationProbabilities.Add(coarseProbabilities); while (coarseLocationProbabilities.Count > numLocationPredictions) { coarseLocationProbabilities.Take(); } //// compute the mode of the array //var groups = coarseLocationPredictions.GroupBy(v => v); //int maxCount = groups.Max(g => g.Count()); //coarseLocation = groups.First(g => g.Count() == maxCount).Key; // sum up the probabilities Dictionary <string, float> totalProbabilities = new Dictionary <string, float>(); foreach (Dictionary <string, float> probabilities in coarseLocationProbabilities) { foreach (string key in probabilities.Keys) { if (!totalProbabilities.ContainsKey(key)) { totalProbabilities[key] = 0; } totalProbabilities[key] += probabilities[key] / numLocationPredictions; } } float maxProb = 0; foreach (string key in totalProbabilities.Keys) { if (totalProbabilities[key] > maxProb) { maxProb = totalProbabilities[key]; coarseLocation = key; coarseProbability = maxProb; } } coarseLocationPredictions.Add(coarseLocation); gestureCoarseLocationPredictions.Add(coarseLocation); while (coarseLocationPredictions.Count > numLocationPredictions) { coarseLocationPredictions.Take(); } if (!Properties.Settings.Default.CoarseOnly) { bool foundFeatureMatch = false; Dictionary <string, float> fineProbabilities = new Dictionary <string, float>(); predictedFineLocation = Localization.Instance.PredictFineLocation(currTemplate, out foundFeatureMatch, out fineProbabilities, true, false, false, coarseLocation); //fineLocationPredictions.Add(predictedFineLocation); //while (fineLocationPredictions.Count > numLocationPredictions) fineLocationPredictions.Take(); fineLocationProbabilities.Add(fineProbabilities); gestureFineLocationProbabilities.Add(fineProbabilities); while (fineLocationProbabilities.Count > numLocationPredictions) { fineLocationProbabilities.Take(); } // compute the mode of the array //var groups = fineLocationPredictions.GroupBy(v => v); //int maxCount = groups.Max(g => g.Count()); //fineLocation = groups.First(g => g.Count() == maxCount).Key; // sum up the probabilities totalProbabilities = new Dictionary <string, float>(); //foreach (Dictionary<string, float> probabilities in fineLocationProbabilities) for (int fineIndex = 0; fineIndex < fineLocationProbabilities.Count; fineIndex++) { Dictionary <string, float> probabilities = fineLocationProbabilities.ToArray()[fineIndex]; // make sure that the fine prediction matches the classes for the coarse prediction if (coarseLocationPredictions.ToArray()[fineIndex] == coarseLocation) { foreach (string key in probabilities.Keys) { if (!totalProbabilities.ContainsKey(key)) { totalProbabilities[key] = 0; } totalProbabilities[key] += probabilities[key] / numLocationPredictions; } } } maxProb = 0; foreach (string key in totalProbabilities.Keys) { if (totalProbabilities[key] > maxProb) { maxProb = totalProbabilities[key]; fineLocation = key; fineProbability = maxProb; } } } Logging.LogLocationEvent(predictedCoarseLocation + " " + predictedFineLocation, coarseLocation + " " + fineLocation); gestureFocusWeights.Add(focus); FPS.Instance("matching").Update(); hasUpdate = true; } if (autoTrainLocation && (coarseLocation != coarseLocationToTrain || fineLocation != fineLocationToTrain) && Monitor.TryEnter(trainingLock)) { trainingForm.Training = true; ImageTemplate newTemplate = CopyTemplate(currTemplate); Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = true; })); if (Properties.Settings.Default.EnableSoundEffects) { captureSound.Play(); Logging.LogAudioEvent("capture", false); } AddTemplate(newTemplate); Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]); Thread.Sleep(100); //if (Properties.Settings.Default.EnableSoundEffects) { beepSound.Play(); Logging.LogAudioEvent("beep", false); } trainingForm.Training = false; Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = false; })); Monitor.Exit(trainingLock); return; } if ((DateTime.Now - touchStart).TotalMilliseconds > Properties.Settings.Default.HoverTimeThreshold) { hovering = true; //if (hoverCoarseLocation == null || coarseLocation == hoverCoarseLocation) // make sure we have the same coarse location, to help prevent jumping around { hoverCoarseLocation = coarseLocation; if (hoverFineLocation == null || fineLocation != hoverFineLocation) // make sure we haven't reported the fine location already { hoverFineLocation = fineLocation; Debug.WriteLine(hoverCoarseLocation + " " + hoverFineLocation); Invoke(new MethodInvoker(delegate { GesturePredictionLabel.Text = "Hover"; })); if (Properties.Settings.Default.EnableSpeechOutput) { if (Properties.Settings.Default.EnableApplicationDemos) { string actionResult = GestureActionMap.PerformAction("Hover", coarseLocation, fineLocation, Properties.Settings.Default.GestureMode, Properties.Settings.Default.FixedApplicationResponses); if (actionResult != null && actionResult.Length > 0) { speech.SpeakAsyncCancelAll(); speech.SelectVoice(MenuVoice); speech.SpeakAsync(actionResult); Logging.LogAudioEvent(actionResult); } } else { speech.SpeakAsyncCancelAll(); speech.SelectVoice(MenuVoice); speech.SpeakAsync("Hover " + coarseLocation + " " + fineLocation); Logging.LogAudioEvent("Hover " + coarseLocation + " " + fineLocation); } } } } } } finally { Monitor.Exit(recognitionLock); } } //LBP.GetInstance(frame.Image.Size).GetHistogram(frame); Invoke(new MethodInvoker(delegate { Display.Image = frame.Image.Bitmap; if (hasUpdate) { CoarsePredictionLabel.Text = coarseLocation; CoarseProbabilityLabel.Text = " (response = " + (coarseProbability * 100).ToString("0.0") + ")"; //Debug.WriteLine(coarseLocation); FinePredictionLabel.Text = fineLocation; FineProbabilityLabel.Text = " (response = " + (fineProbability * 100).ToString("0.0") + ")"; //Debug.WriteLine(fineLocation); } //else if(!touchDown) //{ // CoarsePredictionLabel.Text = ""; // CoarseProbabilityLabel.Text = ""; // FinePredictionLabel.Text = ""; // FineProbabilityLabel.Text = ""; //} //focus = (int)(focus / 100) * 100; Text = FPS.Camera.Average.ToString("0") + " fps camera / " + (Sensors.Instance.IsConnected ? FPS.Sensors.Average.ToString("0") + " fps sensors / " + FPS.Instance("processing").Average.ToString("0") + " fps processing / " + FPS.Instance("matching").Average.ToString("0") + " fps matching" : "Waiting for Sensors") /*+ " / focus = " + focus.ToString("0")*/; })); } catch { } }); }