public Mat Plot(ScreenProperties screen, EyeGazeCalibrater calib) { var pre = new Point3D[Data.Count]; using (var plot = Plot(screen)) { int i = 0; foreach (var item in Data) { pre[i] = item.Value.Face.GazeInfo.Vector; calib.Apply(item.Value.Face, screen); i++; } using (var newPlot = Plot(screen)) { i = 0; foreach (var item in Data) { item.Value.Face.GazeInfo.Vector = pre[i]; item.Value.Face.GazeInfo.UpdateScreenPoint(item.Value.Face, screen); i++; } var fontsize = Core.Cv.GetFontSize(HersheyFonts.HersheyComplexSmall, 0.5); var show = MatTool.New(new Size(plot.Width * 2, plot.Height), MatType.CV_8UC3); Core.Cv.DrawMatAlpha(show, plot, new Point(0, 0)); Core.Cv.DrawMatAlpha(show, newPlot, new Point(plot.Width, 0)); return(show); } } }
public void Apply(FaceRect face, ScreenProperties screen) { if (!IsCalibrating) { Engine.Apply(face, screen); } }
public void UpdateScreenPoint(FaceRect face, ScreenProperties screen) { ScreenPoint = face.SolveRayScreenVector(Vector, screen); if (ClipToBound) { ScreenPoint.X = Util.Clamp(ScreenPoint.X, 0, screen.PixelSize.Width); ScreenPoint.Y = Util.Clamp(ScreenPoint.Y, 0, screen.PixelSize.Height); } }
public Point SolveRayScreenRodrigues(double[] rod, ScreenProperties properties) { double[,] rotMat; Core.Cv.Rodrigues(rod, out rotMat); var rotMatMat = CreateMatrix.DenseOfArray(rotMat); var tempVec = CreateVector.Dense(new double[] { 0, 0, -1 }) * rotMatMat; return(SolveRayScreenVector(new Point3D(tempVec.ToArray()), properties)); }
public EyeGazeDetector(ScreenProperties screen) { ScreenProperties = screen ?? throw new ArgumentNullException("screen properites"); sess = new Session(ModelGraphSingle); sessEx = new Session(ModelGraphExtend); sessFace = new Session(ModelGraphFace); sessFaceMobile = new Session(ModelGraphFaceMobile); sessFaceV2 = new Session(ModelGraphFaceV2); sessFaceV2Mobile = new Session(ModelGraphFaceV2Mobile); Calibrator = new EyeGazeCalibrater(); }
public void Start(ScreenProperties screen, bool train = true) { if (IsStarted || (calibTask != null && (!calibTask.IsCanceled && !calibTask.IsCompleted && !calibTask.IsFaulted))) { Logger.Throw("Already started"); } IsStarted = true; IsCalibrating = true; tokenSource = new CancellationTokenSource(); calibTask = Task.Factory.StartNew(() => CalibProc(screen, train)); }
public Point3D SolveLookScreenVector(Point scrPt, ScreenProperties properties) { var unitPermm = UnitPerMM; Point3D point3d = properties.ToCameraCoordinate(unitPermm, scrPt); point3d = point3d - new Point3D(LandmarkTransformVector); Vector <double> ptVec = CreateVector.Dense(point3d.ToArray()); ptVec = ptVec / ptVec.L2Norm(); return(new Point3D(ptVec.ToArray())); }
public Point SolveRayScreenVector(Point3D vec, ScreenProperties properties) { var unitPermm = UnitPerMM; var tempVec = CreateVector.Dense(vec.ToArray()); var tempScale = Math.Abs(LandmarkTransformVector[2] / tempVec[2]); tempVec = tempVec * tempScale; tempVec = tempVec + CreateVector.DenseOfArray(LandmarkTransformVector); if (tempVec[2] > 0.001) { Logger.Error("Wrong Vector try to solve"); } var tempScr = properties.ToScreenCoordinate(unitPermm, new Point3D(tempVec.ToArray())); //tempScr.X = properties.PixelSize.Width - Util.FixZero(tempScr.X); //tempScr.Y = -Util.FixZero(tempScr.Y); tempScr.X = Util.FixZero(tempScr.X); tempScr.Y = Util.FixZero(tempScr.Y); return(tempScr); }
public double[] SolveLookScreenRodrigues(Point scrPt, ScreenProperties properties) { Vector <double> ptVec = CreateVector.Dense(SolveLookScreenVector(scrPt, properties).ToArray()); Vector <double> originVec = CreateVector.Dense(new double[] { 0, 0, -1 }); var dotProduct = ptVec.DotProduct(originVec); var crossProduct = Util.CrossProduct(ptVec, originVec); crossProduct = crossProduct / crossProduct.L2Norm(); var theta = Math.Acos(dotProduct); if (theta > Math.PI / 2) { theta = Math.PI - theta; crossProduct = crossProduct * -1; } var rodVec = crossProduct * theta; var rod = rodVec.ToArray(); return(rod); }
public EyeGazeDetector(ScreenProperties screen) { ScreenProperties = screen ?? throw new ArgumentNullException("screen properites"); Calibrator = new EyeGazeCalibrater(); }
public override void Apply(FaceRect face, ScreenProperties screen) { face.GazeInfo.Vector = Apply(face.GazeInfo.Vector); face.GazeInfo.UpdateScreenPoint(face, screen); }
public abstract void Apply(FaceRect face, ScreenProperties screen);
public Mat Plot(ScreenProperties screen) { var errorList = new List <double>(); var errorMM = new List <double>(); var ptList = new List <Point>(); var ptDistList = new List <Point>(); var frameMargin = new Point(12, 12); var frameSize = new Size(480, 480); var frameBackSize = frameSize.Clone(); frameBackSize.Width += frameMargin.Y * 2; frameBackSize.Height += frameMargin.Y * 2; var frame = MatTool.New(frameBackSize, MatType.CV_8UC3); using (Mat m = MatTool.New(frameSize, MatType.CV_8UC3)) { m.DrawRectangle(new Rect(0, 0, m.Width, m.Height), Scalar.BgrWhite, -1); foreach (var item in Data) { var pt = frameSize.Center; var ptDist = pt.Clone(); var key3d = item.Key * (-1 / item.Key.Z); var key = new Point(key3d.X, key3d.Y); pt.X *= key.X; pt.Y *= key.Y; var gazeVec = item.Value.Face.GazeInfo.Vector; ptDist.X *= gazeVec.X; ptDist.Y *= gazeVec.Y; pt += frameSize.Center; ptDist += frameSize.Center; ptList.Add(pt); ptDistList.Add(ptDist); var errorDiff = key - new Point(gazeVec.X, gazeVec.Y); var error = Math.Sqrt(Math.Pow(errorDiff.X, 2) + Math.Pow(errorDiff.Y, 2)); errorList.Add(error); var mmDist = item.Value.Face.LandmarkTransform.Z / item.Value.Face.UnitPerMM; errorMM.Add(Math.Abs(screen.Origin.Z - mmDist) * error / 10); } var errorMax = errorList.Max(); var errorMin = errorList.Min(); var errorAvg = errorList.Average(); var errorMMAvg = errorMM.Average(); for (int i = 0; i < ptList.Count; i++) { var pt = ptList[i]; var ptDist = ptDistList[i]; var error = errorList[i]; var alpha = (error - errorMin) / (errorMax - errorMin); var color = Scalar.Blend(Scalar.BgrBlue, 1 - alpha, Scalar.BgrRed, alpha); m.DrawArrow(pt, ptDist, color, 1, LineTypes.AntiAlias, 0.15); } Core.Cv.DrawText(m, $"Mean error: {errorAvg.ToString("0.000")}\n" + $"Mean error(cm): {errorMMAvg.ToString("0.00")}\n" + $"Mean error(degree): {(Math.Atan(errorAvg) / Math.PI * 180).ToString("0.00")}\n" + $"Samples: {errorList.Count}", new Point(10, 25), HersheyFonts.HersheyComplexSmall, 0.5, Scalar.BgrBlack, 1); frame.DrawRectangle(new Rect(0, 0, frame.Width, frame.Height), new Scalar(64, 64, 64), -1); Core.Cv.DrawMatAlpha(frame, m, frameMargin); } return(frame); }
private void CalibProc(ScreenProperties screen, bool train) { var token = tokenSource.Token; if (token.IsCancellationRequested) { return; } var labelResultDict = new Dictionary <Point3D, CalibratingPushData>(); var calibed = new bool[GridWidth * GridHeight]; CalibrateBegin?.Invoke(this, null); for (int i = 0; i < calibed.Length; i++) { if (token.IsCancellationRequested) { return; } int targetIndex = 1; while (true) { var ind = Random.R.NextInt(0, calibed.Length); ind = Math.Min(ind, calibed.Length - 1); if (!calibed[ind]) { targetIndex = ind; break; } } double x = (double)(targetIndex % GridWidth) / (GridWidth - 1); x = x * screen.PixelSize.Width; double y = Math.Floor((double)targetIndex / GridWidth) / (GridHeight - 1); y = y * screen.PixelSize.Height; var targetPoint = new Point(x, y); var calibPercent = (double)i / calibed.Length; Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.Point, targetPoint, calibPercent)); Task.Delay((int)Interval).Wait(); if (token.IsCancellationRequested) { return; } Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.Wait, targetPoint, calibPercent)); Task.Delay((int)WaitInterval).Wait(); if (token.IsCancellationRequested) { return; } for (int sampling = 0; sampling < SampleCount; sampling++) { var samplePercent = calibPercent + ((double)(sampling + 1) / SampleCount) * (1.0 / calibed.Length); Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.SampleWait, targetPoint, samplePercent)); Task.Delay((int)SampleWaitInterval).Wait(); if (token.IsCancellationRequested) { return; } if (lastData == null || lastData.Face.GazeInfo == null) { Logger.Error("Data is not sented... Maybe machine is too slow."); while (lastData == null || lastData.Face.GazeInfo == null) { if (token.IsCancellationRequested) { return; } Task.Delay(500).Wait(); Logger.Error("Gaze is not captured"); } } Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.Sample, targetPoint, samplePercent)); Task.Delay((int)SampleInterval).Wait(); if (token.IsCancellationRequested) { return; } var targetVec = lastData.Face.SolveLookScreenVector(targetPoint, screen); labelResultDict.Add(targetVec, lastData); Logger.Log(this, $"Calibrating {targetPoint} ({i + 1}/{calibed.Length}) [{sampling + 1}/{SampleCount}] - {targetVec} : {lastData.Face.GazeInfo.Vector}"); } calibed[targetIndex] = true; } Engine.SetData(labelResultDict); if (train) { Engine.Train(); } IsCalibrating = false; Logger.Log(this, "Calibrated"); Calibrated?.Invoke(this, new CalibratedArgs(labelResultDict, token)); IsStarted = false; }