private void StartWebCam(BackgroundWorker worker = null)
{
if (cap == null)
{
cap = new VideoCapture(0);
}
if (!cap.Open(0))
{
return;
}
OpenCvSharp.Cv2.NamedWindow("Video", WindowMode.AutoSize);
int cnt = 0;
Mat frame = new Mat();
EyePoints rightEye = new EyePoints(true);
EyePoints leftEye = new EyePoints(false);
IsRunning = true;
while (IsRunning)
{
bool result = cap.Read(frame);
if (!result)
{
worker.CancelAsync();
IsRunning = false;
}
if (frame != null && (frame.Rows * frame.Cols > 0))
{
cnt++;
if (cnt % frameskip == 0)
{
FrameQueue.Enqueue(frame);
cnt = 0;
}
}
while (FrameQueue.Count > 0)
{
Mat que = FrameQueue.Dequeue();
Rect[] faces = GetFaces(que, 1);
for (int i = 0; i < faces.Length; i++)
{
//GetFaceInRect(faces[i], que, i);
Scalar eyecolor = new Scalar(0, 0, 255);
Array2D <byte> gray = ConvertMatToDlib2DArray(que);
FullObjectDetection landmarks = predictor.Detect(gray, ConvertToDlib(faces[i]));
InitializeEyes(landmarks, leftEye, rightEye);
//DrawEye(que, landmarks, leftEye);
//DrawEye(que, landmarks, rightEye);
Rect leftboundingBox = BoundingBoxAroundEye(leftEye, 0);
DrawRect(que, leftboundingBox);
OpenCvSharp.Point centerOfLeftEye = DetectCenterOfEye(que, leftboundingBox);
centerOfLeftEye.X += leftboundingBox.X;
Rect rightboundingBox = BoundingBoxAroundEye(rightEye, 0);
DrawRect(que, rightboundingBox);
OpenCvSharp.Point centerOfRightEye = DetectCenterOfEye(que, rightboundingBox);
centerOfRightEye.X += rightboundingBox.X;
EyeDirection leftEyeDirection = leftEye.GetEyePosition(centerOfLeftEye);
EyeDirection rightEyeDirection = rightEye.GetEyePosition(centerOfRightEye);
EyeDirection eyeDirection = EyeDirection.unknown;
if (leftEyeDirection == EyeDirection.center || rightEyeDirection == EyeDirection.center)
{
eyeDirection = EyeDirection.center;
}
else if (leftEyeDirection == EyeDirection.left)
{
eyeDirection = EyeDirection.left;
}
else if (rightEyeDirection == EyeDirection.right)
{
eyeDirection = EyeDirection.right;
}
OpenCvSharp.Point position = new OpenCvSharp.Point(50, 50);
Cv2.PutText(img: que, text: eyeDirection.ToDisplay(), org: position, fontFace: HersheyFonts.HersheySimplex, fontScale: 2, new Scalar(0, 0, 255));
}
//BitmapImage bmi = ConvertToBMI(frame, cnt, "D:/junk/TestCamImages");
if (worker != null)
{
//worker.ReportProgress(cnt, bmi);
try
{
OpenCvSharp.Cv2.ImShow("Video", que);
int key = Cv2.WaitKey(10); // as in 10 milliseconds
if (key == 27)
{
worker.CancelAsync();
IsRunning = false;
}
}
catch (Exception ex)
{
string msg = ex.Message;
}
}
if (worker.CancellationPending)
{
Cv2.DestroyWindow("Video");
break;
}
}
}
}
private void AnalyzeDepthImage()
{
while (m_IsRunning)
{
if (m_Queue != null)
{
DepthFrame f;
if (m_Queue.Dequeue(out f))
{
using (f)
{
var o_Result = new PlaneResult();
o_Result.IsPlaneReconstructed = true;
var profile = f.GetProfile();
var streamType = profile.Stream;
var streamFormat = profile.Format;
if (streamType == Stream.Depth && streamFormat == Format.Z16)
{
m_Intrinsics = profile.GetIntrinsics();
m_Roi = setROI();
var data = f.Data;
int width = f.Width;
int height = f.Height;
var pixels = f.Pixels; //new short[width * height];
//System.Runtime.InteropServices.Marshal.Copy(data, pixels, 0, width * height);
List <Vector3> roiPixels = new List <Vector3>();
// converting pixels to point in 3d space
for (int y = m_Roi.MinY; y < m_Roi.MaxY; y++)
{
for (int x = m_Roi.MinX; x < m_Roi.MaxX; x++)
{
ushort depthRaw = (ushort)pixels[y * width + x];
// depthRaw if 0 means there's no depth, the pixel is at depth zero just like any other pixel in a photo
if (depthRaw != 0)
{
float[] pixel = { x, y };
Vector3 point = new Vector3();
var distance = depthRaw * m_Units;
projectPixelToPoint(ref point, m_Intrinsics, pixel, distance);
roiPixels.Add(new Vector3(point.X, -point.Y, point.Z));
}
}
}
if (roiPixels.Count < 3)
{
continue;
}
roiPixels.Reverse();
var plane = planeFromPoints(roiPixels);
// The points in RoI don't span a valid plane which is kind of useless
if (plane == new Plane(0, 0, 0, 0))
{
continue;
}
// Vector3 planeFitPivot = approximateIntersection(plane, m_Intrinsics.width / 2f, m_Intrinsics.height / 2f, 0f, 1000f);
Vector3[] planeCorners = new Vector3[4];
planeCorners[0] = approximateIntersection(plane, m_Roi.MinX, m_Roi.MinY, 0f, 1000f);
planeCorners[1] = approximateIntersection(plane, m_Roi.MaxX, m_Roi.MinY, 0f, 1000f);
planeCorners[2] = approximateIntersection(plane, m_Roi.MaxX, m_Roi.MaxY, 0f, 1000f);
planeCorners[3] = approximateIntersection(plane, m_Roi.MinX, m_Roi.MaxY, 0f, 1000f);
o_Result.P = plane;
o_Result.PlaneCorners = planeCorners;
if (!isPlaneValid(o_Result))
{
o_Result.PlaneCorners = null;
o_Result.IsPlaneReconstructed = false;
}
else
{
setAngleRelativeToCameraForward(ref o_Result);
}
// Resulting plance distance from the camera
o_Result.Distance = -plane.D * 1000;
OnRes?.Invoke(o_Result);
}
}
}
else
{
Thread.Sleep(4);
}
}
}
}
