void Camera_FrameAvailable(VideoFrame frame)
{
if (closing)
{
return;
}
ImageTemplate template = new ImageTemplate(frame.Clone());
FPS.Camera.Update();
Task.Factory.StartNew(() =>
{
try
{
Logging.LogVideoFrame(frame.Image.Bitmap);
float focus = 0;
if (Monitor.TryEnter(preprocessingLock))
{
ImageProcessing.ProcessTemplate(template, false);
focus = ImageProcessing.ImageFocus(template);
Logging.LogFrameProcessed();
currTemplate = template;
FPS.Instance("processing").Update();
Monitor.Exit(preprocessingLock);
}
else
{
return;
}
string coarseLocation = "", fineLocation = "";
float coarseProbability = 0, fineProbability = 0;
bool hasUpdate = false;
string predictedCoarseLocation = "", predictedFineLocation = "";
if (touchDown && !trainingForm.Training && Monitor.TryEnter(recognitionLock))
{
try
{
if (Localization.Instance.GetNumTrainingExamples() > 0)
{
Dictionary <string, float> coarseProbabilities = new Dictionary <string, float>();
predictedCoarseLocation = Localization.Instance.PredictCoarseLocation(currTemplate, out coarseProbabilities);
coarseLocation = predictedCoarseLocation;
if (!Properties.Settings.Default.CoarseOnly)
{
bool foundFeatureMatch = false;
Dictionary <string, float> fineProbabilities = new Dictionary <string, float>();
predictedFineLocation = Localization.Instance.PredictFineLocation(currTemplate, out foundFeatureMatch, out fineProbabilities, true, false, false, coarseLocation);
fineLocation = predictedFineLocation;
}
Logging.LogLocationEvent(predictedCoarseLocation + " " + predictedFineLocation);
FPS.Instance("matching").Update();
hasUpdate = true;
}
if (autoTrainLocation && (coarseLocation != coarseLocationToTrain || fineLocation != fineLocationToTrain) && Monitor.TryEnter(trainingLock))
{
trainingForm.Training = true;
ImageTemplate newTemplate = CopyTemplate(currTemplate);
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = true; }));
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play(); Logging.LogAudioEvent("capture", false);
}
AddTemplate(newTemplate);
Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]);
Thread.Sleep(100);
//if (Properties.Settings.Default.EnableSoundEffects) { beepSound.Play(); Logging.LogAudioEvent("beep", false); }
trainingForm.IsSaved = false;
trainingForm.Training = false;
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = false; }));
Monitor.Exit(trainingLock);
return;
}
}
finally
{
Monitor.Exit(recognitionLock);
}
}
//LBP.GetInstance(frame.Image.Size).GetHistogram(frame);
Invoke(new MethodInvoker(delegate
{
Display.Image = frame.Image.Bitmap;
if (hasUpdate)
{
CoarsePredictionLabel.Text = coarseLocation;
FinePredictionLabel.Text = fineLocation;
}
Text = FPS.Camera.Average.ToString("0") + " fps camera / " + (Sensors.Instance.IsConnected ? FPS.Sensors.Average.ToString("0") + " fps sensors / " + FPS.Instance("processing").Average.ToString("0") + "/" + FPS.Instance("matching").Average.ToString("0") + " fps processing" : "Waiting for Sensors") /*+ " / focus = " + focus.ToString("0")*/;
}));
}
catch { }
});
}
//private static void TesseractDownloadLangFile(String folder, String lang)
//{
// String subfolderName = "tessdata";
// String folderName = System.IO.Path.Combine(folder, subfolderName);
// if (!System.IO.Directory.Exists(folderName))
// {
// System.IO.Directory.CreateDirectory(folderName);
// }
// String dest = System.IO.Path.Combine(folderName, String.Format("{0}.traineddata", lang));
// if (!System.IO.File.Exists(dest))
// using (System.Net.WebClient webclient = new System.Net.WebClient())
// {
// String source =
// String.Format("https://github.com/tesseract-ocr/tessdata/blob/4592b8d453889181e01982d22328b5846765eaad/{0}.traineddata?raw=true", lang);
// Console.WriteLine(String.Format("Downloading file from '{0}' to '{1}'", source, dest));
// webclient.DownloadFile(source, dest);
// Console.WriteLine(String.Format("Download completed"));
// }
//}
//private void InitOcr(String path, String lang, OcrEngineMode mode)
//{
// try
// {
// if (ocr != null)
// {
// ocr.Dispose();
// ocr = null;
// }
// if (String.IsNullOrEmpty(path))
// path = ".";
// TesseractDownloadLangFile(path, lang);
// TesseractDownloadLangFile(path, "osd"); //script orientation detection
// String pathFinal = path.Length == 0 ||
// path.Substring(path.Length - 1, 1).Equals(Path.DirectorySeparatorChar.ToString())
// ? path
// : String.Format("{0}{1}", path, System.IO.Path.DirectorySeparatorChar);
// ocr = new Tesseract(pathFinal, lang, mode);
// }
// catch (System.Net.WebException e)
// {
// ocr = null;
// throw new Exception("Unable to download tesseract lang file. Please check internet connection.", e);
// }
// catch (Exception e)
// {
// ocr = null;
// }
//}
private void Camera_FrameAvailable(Bitmap frame)
{
try
{
if (frame == null)
{
return;
}
FPS.Camera.Update();
if (enableThreshold || invertImage)
{
var img = new Image <Gray, byte>(frame);
if (enableThreshold)
{
//CvInvoke.EqualizeHist(img, img);
if (thresholdMode == 0)
{
CvInvoke.Threshold(img, img, 100, 255, Emgu.CV.CvEnum.ThresholdType.Otsu);
}
else if (thresholdMode == 1)
{
CvInvoke.AdaptiveThreshold(img, img, 255, Emgu.CV.CvEnum.AdaptiveThresholdType.GaussianC, Emgu.CV.CvEnum.ThresholdType.Binary, 31, 15);
}
//CvInvoke.MedianBlur(img, img, 7);
//CvInvoke.Dilate(img, img, null, new Point(-1, -1), 1, Emgu.CV.CvEnum.BorderType.Constant, new MCvScalar(255));
//CvInvoke.Erode(img, img, null, new Point(-1, -1), 1, Emgu.CV.CvEnum.BorderType.Constant, new MCvScalar(255));
}
if (invertImage)
{
img = 255 - img;
}
frame = img.ToBitmap();
}
//else if(gamma != 1)
//{
// var img = new Image<Gray, float>(frame);
// CvInvoke.Pow(img, gamma, img);
// frame = img.ToBitmap();
//}
Bitmap imageCopy = (Bitmap)frame.Clone();
Invoke(new MethodInvoker(delegate
{
try
{
if (Display.Image != null)
{
Display.Image.Dispose();
}
Display.Image = imageCopy;
}
catch (Exception ex) { Debug.WriteLine("Error updating camera image: " + ex.ToString()); }
}));
//var img = new Image<Gray, byte>(frame);
////if (ocr != null)
////{
//// ocr.SetImage(img);
//// ocr.Recognize();
//// var chars = ocr.GetCharacters();
////}
//VectorOfERStat regions = new VectorOfERStat();
//textDetector.Run(img, regions);
//img.Dispose();
double frameTime = enableCamera ? 1.0 / frameRate : 1;
DateTime currTime = DateTime.Now;
if ((currTime - lastFrameSent).TotalSeconds > frameTime && !pauseStreaming)
{
FPS.Instance("Stream").Update();
frame.RotateFlip(RotateFlipType.Rotate180FlipNone);
Bitmap newFrame = new Bitmap((int)(frame.Width * prescale), (int)(frame.Height * prescale));
Graphics g = Graphics.FromImage(newFrame);
g.DrawImage(enableCamera ? frame : (invertImage ? invertedPlaceholderImage : placeholderImage), 0, 0, newFrame.Width, newFrame.Height);
server.SendImage(newFrame);
newFrame.Dispose();
//frame.Dispose();
lastFrameSent = currTime;
Invoke(new MethodInvoker(delegate
{
try
{
InfoLabel.Text = "Camera " + FPS.Camera.Average.ToString("0") + " fps, Stream " + FPS.Instance("Stream").Average.ToString("0") + " fps";
}
catch (Exception ex) { Debug.WriteLine("Error setting info label text: " + ex.ToString()); }
}));
}
}
catch (Exception ex) { Debug.WriteLine("Error processing frame from finger camera: " + ex.ToString()); }
}
void Camera_FrameAvailable(VideoFrame frame)
{
if (closing)
{
return;
}
numLocationPredictions = Properties.Settings.Default.TestingMode.Contains("Training") ? 1 : Properties.Settings.Default.PredictionSmoothing;
ImageTemplate template = new ImageTemplate(frame.Clone());
FPS.Camera.Update();
Task.Factory.StartNew(() =>
{
try
{
Logging.LogVideoFrame(frame.Image.Bitmap);
float focus = 0;
//lock (processingLock)
if (Monitor.TryEnter(preprocessingLock))
{
ImageProcessing.ProcessTemplate(template, false);
focus = ImageProcessing.ImageFocus(template);
Logging.LogFrameProcessed();
currTemplate = template;
FPS.Instance("processing").Update();
Monitor.Exit(preprocessingLock);
}
else
{
return;
}
string coarseLocation = "", fineLocation = "";
float coarseProbability = 0, fineProbability = 0;
bool hasUpdate = false;
string predictedCoarseLocation = "", predictedFineLocation = "";
if (touchDown && !trainingForm.Training && Monitor.TryEnter(recognitionLock))
{
try
{
if (Localization.Instance.GetNumTrainingExamples() > 0)
{
Dictionary <string, float> coarseProbabilities = new Dictionary <string, float>();
predictedCoarseLocation = Localization.Instance.PredictCoarseLocation(currTemplate, out coarseProbabilities);
//coarseLocationPredictions.Add(predictedCoarseLocation);
//while (coarseLocationPredictions.Count > numLocationPredictions) coarseLocationPredictions.Take();
coarseLocationProbabilities.Add(coarseProbabilities);
gestureCoarseLocationProbabilities.Add(coarseProbabilities);
while (coarseLocationProbabilities.Count > numLocationPredictions)
{
coarseLocationProbabilities.Take();
}
//// compute the mode of the array
//var groups = coarseLocationPredictions.GroupBy(v => v);
//int maxCount = groups.Max(g => g.Count());
//coarseLocation = groups.First(g => g.Count() == maxCount).Key;
// sum up the probabilities
Dictionary <string, float> totalProbabilities = new Dictionary <string, float>();
foreach (Dictionary <string, float> probabilities in coarseLocationProbabilities)
{
foreach (string key in probabilities.Keys)
{
if (!totalProbabilities.ContainsKey(key))
{
totalProbabilities[key] = 0;
}
totalProbabilities[key] += probabilities[key] / numLocationPredictions;
}
}
float maxProb = 0;
foreach (string key in totalProbabilities.Keys)
{
if (totalProbabilities[key] > maxProb)
{
maxProb = totalProbabilities[key];
coarseLocation = key;
coarseProbability = maxProb;
}
}
coarseLocationPredictions.Add(coarseLocation);
gestureCoarseLocationPredictions.Add(coarseLocation);
while (coarseLocationPredictions.Count > numLocationPredictions)
{
coarseLocationPredictions.Take();
}
if (!Properties.Settings.Default.CoarseOnly)
{
bool foundFeatureMatch = false;
Dictionary <string, float> fineProbabilities = new Dictionary <string, float>();
predictedFineLocation = Localization.Instance.PredictFineLocation(currTemplate, out foundFeatureMatch, out fineProbabilities, true, false, false, coarseLocation);
//fineLocationPredictions.Add(predictedFineLocation);
//while (fineLocationPredictions.Count > numLocationPredictions) fineLocationPredictions.Take();
fineLocationProbabilities.Add(fineProbabilities);
gestureFineLocationProbabilities.Add(fineProbabilities);
while (fineLocationProbabilities.Count > numLocationPredictions)
{
fineLocationProbabilities.Take();
}
// compute the mode of the array
//var groups = fineLocationPredictions.GroupBy(v => v);
//int maxCount = groups.Max(g => g.Count());
//fineLocation = groups.First(g => g.Count() == maxCount).Key;
// sum up the probabilities
totalProbabilities = new Dictionary <string, float>();
//foreach (Dictionary<string, float> probabilities in fineLocationProbabilities)
for (int fineIndex = 0; fineIndex < fineLocationProbabilities.Count; fineIndex++)
{
Dictionary <string, float> probabilities = fineLocationProbabilities.ToArray()[fineIndex];
// make sure that the fine prediction matches the classes for the coarse prediction
if (coarseLocationPredictions.ToArray()[fineIndex] == coarseLocation)
{
foreach (string key in probabilities.Keys)
{
if (!totalProbabilities.ContainsKey(key))
{
totalProbabilities[key] = 0;
}
totalProbabilities[key] += probabilities[key] / numLocationPredictions;
}
}
}
maxProb = 0;
foreach (string key in totalProbabilities.Keys)
{
if (totalProbabilities[key] > maxProb)
{
maxProb = totalProbabilities[key];
fineLocation = key;
fineProbability = maxProb;
}
}
}
Logging.LogLocationEvent(predictedCoarseLocation + " " + predictedFineLocation, coarseLocation + " " + fineLocation);
gestureFocusWeights.Add(focus);
FPS.Instance("matching").Update();
hasUpdate = true;
}
if (autoTrainLocation && (coarseLocation != coarseLocationToTrain || fineLocation != fineLocationToTrain) && Monitor.TryEnter(trainingLock))
{
trainingForm.Training = true;
ImageTemplate newTemplate = CopyTemplate(currTemplate);
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = true; }));
if (Properties.Settings.Default.EnableSoundEffects)
{
captureSound.Play(); Logging.LogAudioEvent("capture", false);
}
AddTemplate(newTemplate);
Logging.LogTrainingEvent("Added template: " + newTemplate["coarse"] + " " + newTemplate["fine"]);
Thread.Sleep(100);
//if (Properties.Settings.Default.EnableSoundEffects) { beepSound.Play(); Logging.LogAudioEvent("beep", false); }
trainingForm.Training = false;
Invoke(new MethodInvoker(delegate { TrainingLabel.Visible = false; }));
Monitor.Exit(trainingLock);
return;
}
if ((DateTime.Now - touchStart).TotalMilliseconds > Properties.Settings.Default.HoverTimeThreshold)
{
hovering = true;
//if (hoverCoarseLocation == null || coarseLocation == hoverCoarseLocation) // make sure we have the same coarse location, to help prevent jumping around
{
hoverCoarseLocation = coarseLocation;
if (hoverFineLocation == null || fineLocation != hoverFineLocation) // make sure we haven't reported the fine location already
{
hoverFineLocation = fineLocation;
Debug.WriteLine(hoverCoarseLocation + " " + hoverFineLocation);
Invoke(new MethodInvoker(delegate { GesturePredictionLabel.Text = "Hover"; }));
if (Properties.Settings.Default.EnableSpeechOutput)
{
if (Properties.Settings.Default.EnableApplicationDemos)
{
string actionResult = GestureActionMap.PerformAction("Hover", coarseLocation, fineLocation, Properties.Settings.Default.GestureMode, Properties.Settings.Default.FixedApplicationResponses);
if (actionResult != null && actionResult.Length > 0)
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(MenuVoice);
speech.SpeakAsync(actionResult);
Logging.LogAudioEvent(actionResult);
}
}
else
{
speech.SpeakAsyncCancelAll();
speech.SelectVoice(MenuVoice);
speech.SpeakAsync("Hover " + coarseLocation + " " + fineLocation);
Logging.LogAudioEvent("Hover " + coarseLocation + " " + fineLocation);
}
}
}
}
}
}
finally
{
Monitor.Exit(recognitionLock);
}
}
//LBP.GetInstance(frame.Image.Size).GetHistogram(frame);
Invoke(new MethodInvoker(delegate
{
Display.Image = frame.Image.Bitmap;
if (hasUpdate)
{
CoarsePredictionLabel.Text = coarseLocation;
CoarseProbabilityLabel.Text = " (response = " + (coarseProbability * 100).ToString("0.0") + ")";
//Debug.WriteLine(coarseLocation);
FinePredictionLabel.Text = fineLocation;
FineProbabilityLabel.Text = " (response = " + (fineProbability * 100).ToString("0.0") + ")";
//Debug.WriteLine(fineLocation);
}
//else if(!touchDown)
//{
// CoarsePredictionLabel.Text = "";
// CoarseProbabilityLabel.Text = "";
// FinePredictionLabel.Text = "";
// FineProbabilityLabel.Text = "";
//}
//focus = (int)(focus / 100) * 100;
Text = FPS.Camera.Average.ToString("0") + " fps camera / " + (Sensors.Instance.IsConnected ? FPS.Sensors.Average.ToString("0") + " fps sensors / " + FPS.Instance("processing").Average.ToString("0") + " fps processing / " + FPS.Instance("matching").Average.ToString("0") + " fps matching" : "Waiting for Sensors") /*+ " / focus = " + focus.ToString("0")*/;
}));
}
catch { }
});
}