public Mat Plot(ScreenProperties screen, EyeGazeCalibrater calib)
{
var pre = new Point3D[Data.Count];
using (var plot = Plot(screen))
{
int i = 0;
foreach (var item in Data)
{
pre[i] = item.Value.Face.GazeInfo.Vector;
calib.Apply(item.Value.Face, screen);
i++;
}
using (var newPlot = Plot(screen))
{
i = 0;
foreach (var item in Data)
{
item.Value.Face.GazeInfo.Vector = pre[i];
item.Value.Face.GazeInfo.UpdateScreenPoint(item.Value.Face, screen);
i++;
}
var fontsize = Core.Cv.GetFontSize(HersheyFonts.HersheyComplexSmall, 0.5);
var show = MatTool.New(new Size(plot.Width * 2, plot.Height), MatType.CV_8UC3);
Core.Cv.DrawMatAlpha(show, plot, new Point(0, 0));
Core.Cv.DrawMatAlpha(show, newPlot, new Point(plot.Width, 0));
return(show);
}
}
}
public void Apply(FaceRect face, ScreenProperties screen)
{
if (!IsCalibrating)
{
Engine.Apply(face, screen);
}
}
public void UpdateScreenPoint(FaceRect face, ScreenProperties screen)
{
ScreenPoint = face.SolveRayScreenVector(Vector, screen);
if (ClipToBound)
{
ScreenPoint.X = Util.Clamp(ScreenPoint.X, 0, screen.PixelSize.Width);
ScreenPoint.Y = Util.Clamp(ScreenPoint.Y, 0, screen.PixelSize.Height);
}
}
public Point SolveRayScreenRodrigues(double[] rod, ScreenProperties properties)
{
double[,] rotMat;
Core.Cv.Rodrigues(rod, out rotMat);
var rotMatMat = CreateMatrix.DenseOfArray(rotMat);
var tempVec = CreateVector.Dense(new double[] { 0, 0, -1 }) * rotMatMat;
return(SolveRayScreenVector(new Point3D(tempVec.ToArray()), properties));
}
public EyeGazeDetector(ScreenProperties screen)
{
ScreenProperties = screen ?? throw new ArgumentNullException("screen properites");
sess = new Session(ModelGraphSingle);
sessEx = new Session(ModelGraphExtend);
sessFace = new Session(ModelGraphFace);
sessFaceMobile = new Session(ModelGraphFaceMobile);
sessFaceV2 = new Session(ModelGraphFaceV2);
sessFaceV2Mobile = new Session(ModelGraphFaceV2Mobile);
Calibrator = new EyeGazeCalibrater();
}
public void Start(ScreenProperties screen, bool train = true)
{
if (IsStarted || (calibTask != null && (!calibTask.IsCanceled && !calibTask.IsCompleted && !calibTask.IsFaulted)))
{
Logger.Throw("Already started");
}
IsStarted = true;
IsCalibrating = true;
tokenSource = new CancellationTokenSource();
calibTask = Task.Factory.StartNew(() => CalibProc(screen, train));
}
public Point3D SolveLookScreenVector(Point scrPt, ScreenProperties properties)
{
var unitPermm = UnitPerMM;
Point3D point3d = properties.ToCameraCoordinate(unitPermm, scrPt);
point3d = point3d - new Point3D(LandmarkTransformVector);
Vector <double> ptVec = CreateVector.Dense(point3d.ToArray());
ptVec = ptVec / ptVec.L2Norm();
return(new Point3D(ptVec.ToArray()));
}
public Point SolveRayScreenVector(Point3D vec, ScreenProperties properties)
{
var unitPermm = UnitPerMM;
var tempVec = CreateVector.Dense(vec.ToArray());
var tempScale = Math.Abs(LandmarkTransformVector[2] / tempVec[2]);
tempVec = tempVec * tempScale;
tempVec = tempVec + CreateVector.DenseOfArray(LandmarkTransformVector);
if (tempVec[2] > 0.001)
{
Logger.Error("Wrong Vector try to solve");
}
var tempScr = properties.ToScreenCoordinate(unitPermm, new Point3D(tempVec.ToArray()));
//tempScr.X = properties.PixelSize.Width - Util.FixZero(tempScr.X);
//tempScr.Y = -Util.FixZero(tempScr.Y);
tempScr.X = Util.FixZero(tempScr.X);
tempScr.Y = Util.FixZero(tempScr.Y);
return(tempScr);
}
public double[] SolveLookScreenRodrigues(Point scrPt, ScreenProperties properties)
{
Vector <double> ptVec = CreateVector.Dense(SolveLookScreenVector(scrPt, properties).ToArray());
Vector <double> originVec = CreateVector.Dense(new double[] { 0, 0, -1 });
var dotProduct = ptVec.DotProduct(originVec);
var crossProduct = Util.CrossProduct(ptVec, originVec);
crossProduct = crossProduct / crossProduct.L2Norm();
var theta = Math.Acos(dotProduct);
if (theta > Math.PI / 2)
{
theta = Math.PI - theta;
crossProduct = crossProduct * -1;
}
var rodVec = crossProduct * theta;
var rod = rodVec.ToArray();
return(rod);
}
public EyeGazeDetector(ScreenProperties screen)
{
ScreenProperties = screen ?? throw new ArgumentNullException("screen properites");
Calibrator = new EyeGazeCalibrater();
}
public override void Apply(FaceRect face, ScreenProperties screen)
{
face.GazeInfo.Vector = Apply(face.GazeInfo.Vector);
face.GazeInfo.UpdateScreenPoint(face, screen);
}
public abstract void Apply(FaceRect face, ScreenProperties screen);
public Mat Plot(ScreenProperties screen)
{
var errorList = new List <double>();
var errorMM = new List <double>();
var ptList = new List <Point>();
var ptDistList = new List <Point>();
var frameMargin = new Point(12, 12);
var frameSize = new Size(480, 480);
var frameBackSize = frameSize.Clone();
frameBackSize.Width += frameMargin.Y * 2;
frameBackSize.Height += frameMargin.Y * 2;
var frame = MatTool.New(frameBackSize, MatType.CV_8UC3);
using (Mat m = MatTool.New(frameSize, MatType.CV_8UC3))
{
m.DrawRectangle(new Rect(0, 0, m.Width, m.Height), Scalar.BgrWhite, -1);
foreach (var item in Data)
{
var pt = frameSize.Center;
var ptDist = pt.Clone();
var key3d = item.Key * (-1 / item.Key.Z);
var key = new Point(key3d.X, key3d.Y);
pt.X *= key.X;
pt.Y *= key.Y;
var gazeVec = item.Value.Face.GazeInfo.Vector;
ptDist.X *= gazeVec.X;
ptDist.Y *= gazeVec.Y;
pt += frameSize.Center;
ptDist += frameSize.Center;
ptList.Add(pt);
ptDistList.Add(ptDist);
var errorDiff = key - new Point(gazeVec.X, gazeVec.Y);
var error = Math.Sqrt(Math.Pow(errorDiff.X, 2) + Math.Pow(errorDiff.Y, 2));
errorList.Add(error);
var mmDist = item.Value.Face.LandmarkTransform.Z / item.Value.Face.UnitPerMM;
errorMM.Add(Math.Abs(screen.Origin.Z - mmDist) * error / 10);
}
var errorMax = errorList.Max();
var errorMin = errorList.Min();
var errorAvg = errorList.Average();
var errorMMAvg = errorMM.Average();
for (int i = 0; i < ptList.Count; i++)
{
var pt = ptList[i];
var ptDist = ptDistList[i];
var error = errorList[i];
var alpha = (error - errorMin) / (errorMax - errorMin);
var color = Scalar.Blend(Scalar.BgrBlue, 1 - alpha, Scalar.BgrRed, alpha);
m.DrawArrow(pt, ptDist, color, 1, LineTypes.AntiAlias, 0.15);
}
Core.Cv.DrawText(m,
$"Mean error: {errorAvg.ToString("0.000")}\n" +
$"Mean error(cm): {errorMMAvg.ToString("0.00")}\n" +
$"Mean error(degree): {(Math.Atan(errorAvg) / Math.PI * 180).ToString("0.00")}\n" +
$"Samples: {errorList.Count}",
new Point(10, 25), HersheyFonts.HersheyComplexSmall, 0.5, Scalar.BgrBlack, 1);
frame.DrawRectangle(new Rect(0, 0, frame.Width, frame.Height), new Scalar(64, 64, 64), -1);
Core.Cv.DrawMatAlpha(frame, m, frameMargin);
}
return(frame);
}
private void CalibProc(ScreenProperties screen, bool train)
{
var token = tokenSource.Token;
if (token.IsCancellationRequested)
{
return;
}
var labelResultDict = new Dictionary <Point3D, CalibratingPushData>();
var calibed = new bool[GridWidth * GridHeight];
CalibrateBegin?.Invoke(this, null);
for (int i = 0; i < calibed.Length; i++)
{
if (token.IsCancellationRequested)
{
return;
}
int targetIndex = 1;
while (true)
{
var ind = Random.R.NextInt(0, calibed.Length);
ind = Math.Min(ind, calibed.Length - 1);
if (!calibed[ind])
{
targetIndex = ind;
break;
}
}
double x = (double)(targetIndex % GridWidth) / (GridWidth - 1);
x = x * screen.PixelSize.Width;
double y = Math.Floor((double)targetIndex / GridWidth) / (GridHeight - 1);
y = y * screen.PixelSize.Height;
var targetPoint = new Point(x, y);
var calibPercent = (double)i / calibed.Length;
Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.Point, targetPoint, calibPercent));
Task.Delay((int)Interval).Wait();
if (token.IsCancellationRequested)
{
return;
}
Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.Wait, targetPoint, calibPercent));
Task.Delay((int)WaitInterval).Wait();
if (token.IsCancellationRequested)
{
return;
}
for (int sampling = 0; sampling < SampleCount; sampling++)
{
var samplePercent = calibPercent + ((double)(sampling + 1) / SampleCount) * (1.0 / calibed.Length);
Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.SampleWait, targetPoint, samplePercent));
Task.Delay((int)SampleWaitInterval).Wait();
if (token.IsCancellationRequested)
{
return;
}
if (lastData == null || lastData.Face.GazeInfo == null)
{
Logger.Error("Data is not sented... Maybe machine is too slow.");
while (lastData == null || lastData.Face.GazeInfo == null)
{
if (token.IsCancellationRequested)
{
return;
}
Task.Delay(500).Wait();
Logger.Error("Gaze is not captured");
}
}
Calibarting.Invoke(this, new CalibratingArgs(CalibratingState.Sample, targetPoint, samplePercent));
Task.Delay((int)SampleInterval).Wait();
if (token.IsCancellationRequested)
{
return;
}
var targetVec = lastData.Face.SolveLookScreenVector(targetPoint, screen);
labelResultDict.Add(targetVec, lastData);
Logger.Log(this, $"Calibrating {targetPoint} ({i + 1}/{calibed.Length}) [{sampling + 1}/{SampleCount}] - {targetVec} : {lastData.Face.GazeInfo.Vector}");
}
calibed[targetIndex] = true;
}
Engine.SetData(labelResultDict);
if (train)
{
Engine.Train();
}
IsCalibrating = false;
Logger.Log(this, "Calibrated");
Calibrated?.Invoke(this, new CalibratedArgs(labelResultDict, token));
IsStarted = false;
}