internal override void Next(Face face)
{
if (timer.ElapsedMilliseconds > this.MaxSeconds)
{
this.AbortClick();
this.ResetTimer();
}
else if (face.HasBothEyesOpen)
{
if (timer.ElapsedMilliseconds < this.MinSeconds)
{
this.AboutToAbort(300);
}
else
{
((ClickAction)this.action).Clicks++;
this.ResetTimer();
this.BothEyesOpenState();
}
}
else if (this.IsBlinking(face))
{
this.IsBlinkingState();
}
else if (face.HasBothEyesClosed)
{
this.AboutToAbort(200);
}
else
{
this.AboutToAbort(100);
}
}
internal override void Next(Face face)
{
if (this.timer.ElapsedMilliseconds > 1500)
{
this.timer.Reset();
this.action.SetState<NotAboutToLeftClickState>();
}
else
{
if (face.HasBothEyesOpen)
{
this.action.SetState<InitialLeftDoubleClickState>();
}
else if (face.IsBlinkingLeftEye)
{
this.timer.Reset();
this.action.SetState<AboutToLeftDoubleClickState>();
this.action.GetState<AboutToLeftDoubleClickState>().StartTimer();
}
else
{
this.timer.Reset();
this.action.SetState<NotAboutToLeftClickState>();
}
}
}
internal override void Next(Face face)
{
//Si estuvo menos de 3 segundos con los ojos cerrados
if (face.HasBothEyesClosed)
{
if (timer.ElapsedMilliseconds < 3000)
{
//Sigue igual
this.action.SetState<AboutToActivateSpeechState>();
}
else
{
//Con este estado deberia activar la funcionalidad de dictado
if (!face.IsFake)
this.action.SetState<ShouldActivateSpeechState>();
else
this.action.SetState<NotAboutToActivateSpeechState>();
this.ResetTimer();
}
}
else
{
this.action.SetState<AboutToAbortSpeechState>();
this.action.GetState<AboutToAbortSpeechState>().StartTimer();
}
}
public Word Detect(Face face, Image<Bgr, byte> cleanImage)
{
var loginImagesPaths = new List<string>();
if (Directory.Exists(this.usersPath))
loginImagesPaths = Directory.GetFiles(this.usersPath, "*" + this.gesturesPattern, SearchOption.AllDirectories).ToList();
loginImagesPaths = loginImagesPaths.Where(x => x.Contains(this.loginService.CurrentUser)).ToList();
if (!loginImagesPaths.Any())
{
return new Word(string.Empty);
}
else
{
if (this.shouldReloadImages)
{
this.shouldReloadImages = false;
}
var source = cleanImage.Copy();
source.ROI = new Rectangle
{
X = face.Zone.X,
Y = face.Zone.Y,
Width = face.Zone.Width + face.Zone.Width / 3,
Height = face.Zone.Height + face.Zone.Height / 3
};
for (double scale = 1.2; scale > 0.4; scale = scale - 0.1)
{
foreach (var loginPath in loginImagesPaths)
{
var template = new Image<Bgr, byte>(loginPath);
var path = new String(loginPath.ToArray());
Image<Gray, float> result = source.Convert<Gray, byte>().SmoothGaussian(5)
.MatchTemplate(template.Convert<Gray, byte>()
.Resize(scale, Emgu.CV.CvEnum.INTER.CV_INTER_LINEAR).SmoothGaussian(5),
Emgu.CV.CvEnum.TM_TYPE.CV_TM_CCOEFF_NORMED);
double[] minValues, maxValues;
Point[] minLocations, maxLocations;
result.MinMax(out minValues, out maxValues, out minLocations, out maxLocations);
if (maxValues[0] > 0.9)
{
var newPath = path.Replace("\\", "|");
var splittedPath = newPath.Split('|');
if (face.Mouth.IsEmpty)
face.Mouth = new Mouth();
face.Mouth.Word = new Word(splittedPath[splittedPath.Length - 2]);
return face.Mouth.Word;
}
}
}
}
return new Word(string.Empty);
}
internal override void Process(Face detectedFace)
{
this.deactivateAction.NextState(detectedFace);
if (this.deactivateAction.ShouldBeExecuted())
{
this.speechProcessor.Finish();
this.cursorLoopProcessor.Finish();
}
}
internal override void Next(Face face)
{
if (face.HasBothEyesOpen)
{
this.SetBothEyesOpenState();
}
else
{
this.SetNotBothEyesOpenState();
}
}
internal override void Next(Face face)
{
if (!face.HasBothEyesClosed)
{
this.action.SetState<InitialActivateSpeechState>();
}
else
{
this.action.SetState<NotAboutToActivateSpeechState>();
}
}
internal override void Next(Face face)
{
if (face.IsEmpty)
{
this.action.GetState<InitialDeactivateState>().StartTimer();
this.action.SetState<InitialDeactivateState>();
}
else
{
this.action.SetState<NotAboutToDeactivateState>();
}
}
internal override void Next(Face face)
{
//Si estuvo menos de 3 segundos con los ojos cerrados
if (!face.HasBothEyesClosed)
{
this.action.SetState<InitialActivateSpeechState>();
}
else
{
this.action.GetState<AboutToActivateSpeechState>().StartTimer();
this.action.SetState<AboutToActivateSpeechState>();
}
}
//Implement priority
public FaceDetector(EyeDetector eyeDetector)
{
this.eyeDetector = eyeDetector;
this.lastDetected = new Face();
this.notDetectedTimer = new Stopwatch();
this.facePriority = new LifoAlgorithm<BaseFaceDetector>();
this.frontalFacePriorityItem = new FaceDetectorPriority(new FrontalFaceDetector());
this.profileFacePriorityItem = new FaceDetectorPriority(new ProfileFaceDetector());
this.rotatedFacePriorityItem = new FaceDetectorPriority(new RotatedFaceDetector());
this.facePriority.AddAlgorithmItem(this.frontalFacePriorityItem);
this.facePriority.AddAlgorithmItem(this.profileFacePriorityItem);
this.facePriority.AddAlgorithmItem(this.rotatedFacePriorityItem);
}
internal override void Process(Face detectedFace)
{
foreach (var action in this.cursorActions)
{
action.NextState(detectedFace);
}
foreach (var action in this.cursorActions)
{
if (action.ShouldBeExecuted())
{
action.Execute();
}
}
}
internal override void Next(Face face)
{
if (this.timer.ElapsedMilliseconds < 500)
{
if (face.HasBothEyesClosed)
{
this.action.SetState<AboutToActivateSpeechState>();
this.ResetTimer();
}
}
else
{
this.action.SetState<NotAboutToActivateSpeechState>();
}
}
internal override void Next(Face face)
{
if (this.IsBlinkingEye(face))
{
this.SetIsBlinkingEyeState().StartTimer();
}
else if (face.HasBothEyesOpen)
{
this.SetHasBothEyesOpenState();
}
else
{
this.SetElseState();
}
}
internal override void Next(Face face)
{
if (this.timer.ElapsedMilliseconds < this.MaxErrorMilliseconds)
{
if (this.IsBlinking(face))
{
this.AboutToClick();
this.ResetTimer();
}
}
else
{
((ClickAction)this.action).FakeClicks++;
this.AbortClick();
}
}
public void ShowOriginalImage(Image<Bgr, byte> drawedImage, Face face)
{
var imageToShow = drawedImage.Copy();
//if (!face.IsEmpty)
//{
// imageToShow.ROI = face.Zone;
//}
imageToShow = imageToShow.Flip(FLIP.HORIZONTAL);
this.OriginalImageViewer.Image = imageToShow.Resize(this.OriginalImageViewer.Width, this.OriginalImageViewer.Height, INTER.CV_INTER_LINEAR);
this.SetPoint(new Point()
{
X = Cursor.Position.X + 12,
Y = Cursor.Position.Y + 12
});
}
internal override void Next(Face face)
{
if (face.IsEmpty)
{
if (timer.ElapsedMilliseconds < 3000)
{
this.action.SetState<InitialDeactivateState>();
}
else
{
this.ResetTimer();
this.action.SetState<ShouldDeactivateState>();
}
}
else
{
this.ResetTimer();
this.action.SetState<NotAboutToDeactivateState>();
}
}
public virtual IEnumerable<Face> DetectFaces(Image<Bgr, byte> image)
{
var result = new List<Face>();
var factor = image.Width / this.WidthToReduce;
var reducedImage = image.Resize(image.Width / factor,
image.Height / factor,
INTER.CV_INTER_LINEAR);
reducedImage = this.AfterReducingImage(reducedImage);
Image<Gray, Byte> grayImage = reducedImage.Convert<Gray, byte>();
var faces = this.GetFacesVector(grayImage);
foreach (var face in faces)
{
var detectedFace = new Face();
detectedFace.IsFake = false;
detectedFace.Zone =
new Rectangle(
new Point(
face.rect.X * factor,
face.rect.Y * factor),
new Size(
face.rect.Width * factor,
face.rect.Height * factor));
detectedFace.Image = image;
result.Add(detectedFace);
}
return result;
}
internal void Process(Face detectedFace)
{
this.deactivateSpeechAction.NextState(detectedFace);
if (this.deactivateSpeechAction.ShouldBeExecuted())
{
this.speechProcessor.Finish();
}
this.activateSpeechAction.NextState(detectedFace);
if (this.activateSpeechAction.ShouldBeExecuted())
{
if (!this.speechProcessor.IsStarted())
{
this.speechProcessor.Start();
}
else
{
this.speechProcessor.Finish();
}
}
}
private void Show(Image<Bgr, byte> image, Face face)
{
this.mainForm.ShowOriginalImage(image.Flip(FLIP.HORIZONTAL), face);
this.trainBox.ShowOriginalImage(image, face);
}
public Face DetectFace(Image<Bgr, byte> image)
{
Face detectedFace = new Face();
BaseFaceDetector faceDetector = this.facePriority.Next();
detectedFace = this.GetCentered(image, faceDetector.DetectFaces(image));
if (!detectedFace.IsEmpty)
{
detectedFace.IsOuttaControl = false;
this.notDetectedTimer.Reset();
if (detectedFace.IsFrontal && (this.lastFrontalDetected == null || this.lastFrontalDetected.IsIntoControlZone))
{
this.eyeDetector.DetectRightEye(detectedFace);
this.eyeDetector.DetectLeftEye(detectedFace);
}
detectedFace.Convert();
if (detectedFace.IsFrontal)
{
this.lastFrontalDetected = detectedFace;
if (detectedFace.IsIntoControlZone)
{
this.lastDetected = detectedFace;
this.facePriority.SetFirst(this.frontalFacePriorityItem);
if (this.lastNotFrontalDetected != null)
{
this.lastNotFrontalDetected.IsRightProfile = detectedFace.Center.X < detectedFace.Image.Center().X;
this.lastNotFrontalDetected.IsLeftProfile = !this.lastDetected.IsRightProfile;
this.lastNotFrontalDetected.IsRightRotated = false;
this.lastNotFrontalDetected.IsLeftRotated = false;
}
}
else
{
this.facePriority.SetFirst(this.profileFacePriorityItem);
this.facePriority.SetSecond(this.rotatedFacePriorityItem);
var oldDetectedFace = detectedFace;
this.lastDetected = detectedFace = this.lastNotFrontalDetected ?? detectedFace;
if (this.lastDetected.IsProfile)
{
this.lastDetected.IsRightProfile = oldDetectedFace.Center.X < oldDetectedFace.Image.Center().X;
this.lastDetected.IsLeftProfile = !this.lastDetected.IsRightProfile;
}
if (this.lastDetected.IsRotated)
{
this.lastDetected.IsRightRotated = !(oldDetectedFace.Center.X < oldDetectedFace.Image.Center().X);
this.lastDetected.IsLeftRotated = !this.lastDetected.IsRightProfile;
}
detectedFace.IsOuttaControl = oldDetectedFace.IsZoneOutOfControl;
detectedFace.ReplacedZone = oldDetectedFace.Zone;
}
}
else
{
this.lastDetected = this.lastNotFrontalDetected = detectedFace;
}
if (detectedFace.IsProfile)
{
this.lastDetected = this.lastNotFrontalDetected = detectedFace;
if (detectedFace.IsIntoControlZone)
{
this.facePriority.SetFirst(this.frontalFacePriorityItem);
this.facePriority.SetSecond(this.profileFacePriorityItem);
//this.facePriority.SetSecond(this.rotatedFacePriorityItem);
//detectedFace.IsOuttaControl = false;
}
else
{
this.lastDetected.IsOuttaControl = this.lastDetected.IsZoneOutOfControl;
this.lastDetected.ReplacedZone = this.lastDetected.Zone;
this.facePriority.SetFirst(this.profileFacePriorityItem);
this.facePriority.SetSecond(this.rotatedFacePriorityItem);
}
}
if (detectedFace.IsRotated)
{
this.lastDetected = this.lastNotFrontalDetected = detectedFace;
if (detectedFace.IsIntoControlZone)
{
//this.facePriority.SetFirst(this.rotatedFacePriorityItem);
//this.facePriority.SetSecond(this.profileFacePriorityItem);
this.facePriority.SetFirst(this.frontalFacePriorityItem);
this.facePriority.SetSecond(this.rotatedFacePriorityItem);
//detectedFace.IsOuttaControl = false;
}
else
{
this.lastDetected.IsOuttaControl = this.lastDetected.IsZoneOutOfControl;
this.lastDetected.ReplacedZone = this.lastDetected.Zone;
this.facePriority.SetFirst(this.rotatedFacePriorityItem);
this.facePriority.SetSecond(this.profileFacePriorityItem);
}
}
}
else
{
if (!this.notDetectedTimer.IsRunning)
{
if (!this.lastDetected.IsEmpty)
{
this.lastDetected.IsFake = true;
}
this.notDetectedTimer.Start();
}
if (this.notDetectedTimer.ElapsedMilliseconds < 5000)
{
return this.lastDetected;
}
else
{
this.notDetectedTimer.Stop();
return new Face();
}
}
return detectedFace;
}
public void ShowOriginalImage(Image<Bgr, byte> drawedImage, Face face)
{
this.OriginalImageViewer.Image = drawedImage.Resize(this.OriginalImageViewer.Width, this.OriginalImageViewer.Height, INTER.CV_INTER_LINEAR);
//this.trainBox.ShowOriginalImage(drawedImage, face);
}
protected override bool IsBlinking(Face face)
{
return face.IsBlinkingRightEye;
}
protected override bool IsBlinkingEye(Face face)
{
return face.IsBlinking;
}
protected abstract bool IsBlinkingEye(Face face);
public void DetectRightEye(Face face)
{
var result = new List<Eye>();
face.Image.ROI = face.Zone;
var reducedImage = face.Image.Copy();
Rectangle roi = new Rectangle();
roi.Width = reducedImage.Width / 2;
roi.Height = reducedImage.Height / 2;
roi.Y = reducedImage.ROI.Y + reducedImage.Height / 8;
reducedImage.ROI = roi;
face.Image.ROI = new Rectangle();
Image<Gray, Byte> grayImage = reducedImage.Convert<Gray, byte>();
var eyes = this.DetectVarious(grayImage);
foreach (var eye in eyes)
{
var detectedEye = new Eye();
detectedEye.Zone =
new Rectangle(
new Point(
eye.rect.X + face.Zone.X,
eye.rect.Y + face.Zone.Y + roi.Y),
new Size(
eye.rect.Width,
eye.rect.Height));
detectedEye.Image = face.Image;
//si el ojo no esta en la frente lo agregamos
if (!(detectedEye.Zone.Y + detectedEye.Zone.Height / 2 < face.Zone.Y + face.Zone.Height / 4))
{
//si el ojo no esta en la boca lo agregamos
if (!(detectedEye.Zone.Y + detectedEye.Zone.Height / 2 > face.Zone.Y + face.Zone.Height / 2))
{
//si el ojo no esta fuera de la cara lo agregamos
if (!(detectedEye.Zone.X + detectedEye.Zone.Width / 2 < face.Zone.X + face.Zone.Width / 6))
{
result.Add(detectedEye);
break;
}
}
}
}
var rightEye = result.FirstOrDefault();
face.RightEye = rightEye ?? new Eye();
face.RightEye.Image = face.Image;
}
internal void ProcessImages(object sender, EventArgs e)
{
this.currentImage = webCam.QueryFrame();
this.currentImageClean = this.currentImage.Copy();
this.currentDetectedFace = this.faceDetector.DetectFace(currentImage);
if (!currentDetectedFace.IsEmpty)
{
if (this.loginService.IsLoggedIn())
{
if (!currentDetectedFace.IsFake && !currentDetectedFace.IsOuttaControl)
this.loginService.RestartTimer();
this.deactivateActionProcessor.Process(currentDetectedFace);
this.shouldDrawEyes = false;
if (currentDetectedFace.IsFrontal)
{
var eyesColor = Color.DarkRed;
//Dejamos de mover el puntero del mouse
this.cursorLoopProcessor.Finish();
if (currentDetectedFace.HasEyesCentered && !currentDetectedFace.HasBothEyesClosed)
{
this.lastFrontalFace = this.currentDetectedFace;
eyesColor = Color.Green;
if (!this.clickTimer.IsRunning) { this.clickTimer.Start(); }
if (this.clickTimer.ElapsedMilliseconds > 5000)
{
long allowedFakeClicks = (long)(this.clickTimer.ElapsedMilliseconds / 3 / 1000) + 1;
if (this.cursorActionProcessor.FakeClicks > allowedFakeClicks)
{
if (!this.shouldBlockPrecision)
this.eyeDetector.IncreasePrecision();
this.clickTimer.Restart();
this.cursorActionProcessor.ResetClicksCount();
};
}
this.shouldDrawEyes = true;
}
else
{
if (this.clickTimer.IsRunning)
{
this.clickTimer.Stop();
}
if (this.clickTimer.ElapsedMilliseconds > 30000)
{
this.clickTimer.Restart();
this.cursorActionProcessor.ResetClicksCount();
}
}
this.cursorActionProcessor.Process(currentDetectedFace);
this.activateSpeechAction.NextState(currentDetectedFace);
if (this.activateSpeechAction.ShouldBeExecuted())
{
if (!this.speechProcessor.IsStarted())
{
this.speechProcessor.Start();
}
else
{
this.speechProcessor.Finish();
}
}
if (this.speechProcessor.IsStarted() && !this.isLookingForGestures)
{
this.isLookingForGestures = true;
Task.Run(() =>
{
while (this.speechProcessor.IsStarted() && this.isLookingForGestures)
{
if (this.mouthTimer.ElapsedMilliseconds > 1000)
{
var word = this.gesturesService.Detect(this.currentDetectedFace, this.currentImageClean);
if (!string.IsNullOrWhiteSpace(word.Value))
{
this.keyboardSimulator.PressKey(word);
}
this.mouthTimer.Restart();
}
else
{
if (!this.mouthTimer.IsRunning)
{
this.mouthTimer.Start();
}
}
}
this.isLookingForGestures = false;
});
}
else
{
this.mouthTimer.Stop();
}
//Dibujamos en la imagen la cara detectada
//this.DrawZone(currentDetectedFace, Color.Red);
//Dibujamos la zona de control
this.DrawRectangle(currentDetectedFace.Image, currentDetectedFace.ControlZone, Color.DarkRed);
//Dibujamos el centro de control de la cara
this.DrawCircle(currentDetectedFace.Image, currentDetectedFace.Center, Color.Red);
}
else if (!this.currentDetectedFace.IsOuttaControl)
{
this.cursorLoopProcessor.Start();
this.activateSpeechAction.Reset();
if (currentDetectedFace.IsProfile)
{
if (currentDetectedFace.IsLeftProfile)
{
this.cursorLoopProcessor.MoveCursorToLeft();
//this.DrawZone(currentDetectedFace, Color.AntiqueWhite);
//Dibujamos el centro de control de la cara
this.DrawCircle(currentDetectedFace.Image,
new Point
{
X = currentDetectedFace.ControlZone.X + currentDetectedFace.ControlZone.Width,
Y = currentDetectedFace.Center.Y
},
Color.SkyBlue);
}
else if (currentDetectedFace.IsRightProfile)
{
this.cursorLoopProcessor.MoveCursorToRight();
//this.DrawZone(currentDetectedFace, Color.SkyBlue);
//Dibujamos el centro de control de la cara
this.DrawCircle(currentDetectedFace.Image,
new Point
{
X = currentDetectedFace.ControlZone.X,
Y = currentDetectedFace.Center.Y
},
Color.SkyBlue);
}
this.DrawRectangle(currentDetectedFace.Image, currentDetectedFace.ControlZone, Color.DarkBlue);
}
else
{
if (currentDetectedFace.IsRotated)
{
if (currentDetectedFace.IsLeftRotated)
{
this.cursorLoopProcessor.MoveCursorToBottom();
//this.DrawZone(currentDetectedFace, Color.Violet);
}
else if (currentDetectedFace.IsRightRotated)
{
this.cursorLoopProcessor.MoveCursorToTop();
//this.DrawZone(currentDetectedFace, Color.Violet);
}
//Dibujamos la zona de control
this.DrawRectangle(currentDetectedFace.Image, currentDetectedFace.ControlZone, Color.DarkViolet);
//Dibujamos el centro de control de la cara
this.DrawCircle(currentDetectedFace.Image, currentDetectedFace.Center, Color.Violet);
}
}
}
else
{
//Dejamos de mover el puntero del mouse
this.cursorLoopProcessor.Finish();
this.DrawCircle(currentDetectedFace.Image, currentDetectedFace.ReplacedZone.Center(), Color.Black);
this.DrawRectangle(currentDetectedFace.Image, currentDetectedFace.ControlZone, Color.Black);
}
if (this.shouldDrawEyes)
{
this.DrawZone(currentDetectedFace.RightEye, Color.YellowGreen);
this.DrawZone(currentDetectedFace.LeftEye, Color.YellowGreen);
}
}
else
{
if (!this.currentDetectedFace.IsFake)
this.loginService.Login(this.currentImageClean);
this.cursorActionProcessor.ResetActions();
this.activateSpeechAction.Reset();
this.speechProcessor.Finish();
this.cursorLoopProcessor.Finish();
if (!this.shouldBlockPrecision)
this.ResetPrecision();
}
}
else
{
this.cursorActionProcessor.ResetActions();
this.activateSpeechAction.Reset();
this.deactivateActionProcessor.Process(currentDetectedFace);
}
this.Show(currentImage, currentDetectedFace);
}
private void CalculatePercetages(Face detectedFace)
{
if (detectedFace.HasEyesCentered)
{
this.totalCountValidFaces++;
if (detectedFace.HasBothEyesClosed)
{
this.totalLeftEyeClosed++;
this.totalRightEyeClosed++;
}
else if (detectedFace.IsBlinkingLeftEye)
{
this.totalLeftEyeClosed++;
}
else if (detectedFace.IsBlinkingRightEye)
{
this.totalRightEyeClosed++;
}
}
}
internal override void Next(Face item)
{
this.action.SetState<NotAboutToActivateSpeechState>();
}
internal override void Next(Face face)
{
this.SetNotAboutState();
}
internal abstract void Process(Face detectedFace);
