/* Odświezanie okna z obrazem */ void RefreshWindow(object sender, EventArgs arg) { //Pobieranie ramki image = capture.QueryFrame(); image = image.Flip(FLIP.HORIZONTAL); imageBox1.Image = image; //YCbCr or Bgr(RGB) //Warto zwrócić uwagę na to że Ycc to Y,Cr,Cb a nie Y,Cb,Cr, oraz Bgr to Blue,Green,Red if (radioButton1.Checked) imageGray = image.Resize((double)nupScale.Value, INTER.CV_INTER_CUBIC).Convert<Ycc, Byte>(). InRange(new Ycc((double)nudW1.Value, (double)nudW3.Value, (double)nudW2.Value), new Ycc((double)nudW4.Value, (double)nudW6.Value, (double)nudW5.Value)); else imageGray = image.InRange(new Bgr((double)nudW3.Value, (double)nudW2.Value, (double)nudW1.Value), new Bgr((double)nudW6.Value, (double)nudW5.Value, (double)nudW4.Value)); if (medianCB.Checked) imageGray = imageGray.SmoothMedian((int)nudMedian.Value); //Image<Gray, Byte> sgm = new Image<Gray, Byte>(imageGray.Size); Bitmap bmp = imageGray.ToBitmap(); bc.ProcessImage(bmp); Blob[] blob = bc.GetObjectsInformation(); //one hand version //int iters = bc.ObjectsCount > 1 ? 1 : bc.ObjectsCount; int iters = bc.ObjectsCount > 2 ? 2 : bc.ObjectsCount; if(iters > 1) { //both hands version //lewa reka to ta z prawej strony obrazu (zwierciadlo), nie zakladamy ze user gestykuluje na krzyz, keep it simple blob = blob.Take(2).OrderByDescending(a => a.CenterOfGravity.X).ToArray<Blob>(); } int centerOfGravityLHandX = 0, centerOfGravityLHandY = 0, centerOfGravityRHandX = 0, centerOfGravityRHandY = 0; string[] gestureLabel = new string[2]; int i = 0; for (; i < iters; ++i) { IntPoint minXY, maxXY; PointsCloud.GetBoundingRectangle(bc.GetBlobsEdgePoints(blob[i]), out minXY, out maxXY); Bitmap clonimage = (Bitmap)bmp.Clone(); BitmapData data = bmp.LockBits(new Rectangle(0, 0, imageGray.Width, imageGray.Height), ImageLockMode.ReadWrite, bmp.PixelFormat); Drawing.Rectangle(data, blob[i].Rectangle, Color.White); bmp.UnlockBits(data); int X = maxXY.X, Y = maxXY.Y; int x = minXY.X, y = minXY.Y; observed[0,i] = blob[i].Fullness; /* malinowska kryjaka liczy obwod ze wzoru na prostokąt, nasza liczy piksele krawedziowe */ //Malinowska(i) = (2*bb(3)+2*bb(4))/(2*sqrt(pi*S)) - 1; observed[2,i] = (double)(bc.GetBlobsEdgePoints(blob[i]).Count) / 2 / Math.Sqrt(Math.PI * blob[i].Area) - 1; //MalinowskaZ(i) = 2*sqrt(pi*S)/(2*bb(3)+2*bb(4)); observed[3,i] = 2 * Math.Sqrt(Math.PI * blob[i].Area) / (double)(bc.GetBlobsEdgePoints(blob[i]).Count); int gx = (int)blob[i].CenterOfGravity.X, gy = (int)blob[i].CenterOfGravity.Y; //Sprawdzenie która ręka prawa, a która lewa if (gx > centerOfGravityRHandX) { centerOfGravityLHandX = centerOfGravityRHandX; centerOfGravityLHandY = centerOfGravityRHandY; centerOfGravityRHandX = gx; centerOfGravityRHandY = gy; } else { centerOfGravityLHandX = gx; centerOfGravityLHandY = gy; } double blairsum = 0; int ftx = 0, ftxMax = 0; byte[, ,] dd = imageGray.Data; for (int j = y; j < Y; ++j) { if (ftx > ftxMax) ftxMax = ftx; ftx = 0;//bo moze sie zdazyc ze zliczy wiecej linii naraz, patrz: idealny prostokat for (int k = x; k < X; ++k) { if (dd[j, k, 0] != 0) { ++ftx; blairsum += (k - gx) * (k - gx) + (j - gy) * (j - gy);//distance squared } else { if (ftx > ftxMax) ftxMax = ftx; ftx = 0; } dd[j, k, 0] = 255; } } /* aby policzyc ftyMax trzeba puscic jeszcze jedna petle tak aby wewnetrzna szla po y-kach * ale mozna tez aproksymowac ftYmax przez boundingbox.Y, wtedy * przewidywalem najwieksze rozbieznosci przy skosnych lub dziurawych obiektach; * ale blad byl ponizej procenta, wiec szkoda tracic czas na kolejne O(n*n) int fty = 0, ftyMax = 0; for (int j = x; j < X; ++j) { if (fty > ftyMax) ftyMax = fty; fty = 0; for (int i = y; i < Y; ++i) if (dd[i, j, 0] != 0) ++fty; else { if (fty > ftyMax) ftyMax = fty; fty = 0; } } feret = (double)ftxMax / ftyMax; */ observed[4,i] = (double)ftxMax / (Y-y);//feret observed[1,i] = (double)(blob[i].Area) / Math.Sqrt(2 * Math.PI * blairsum);//blair gestChance[GEST.SLAYER] = dist(slayer, i); gestChance[GEST.THUMBLEFT] = dist(thumbleft, i); gestChance[GEST.THUMBUP] = dist(thumbup, i); gestChance[GEST.SHAKA] = dist(shaka, i); gestChance[GEST.FIST] = dist(fist, i); gestChance[GEST.VICTORY] = dist(victory, i); gestChance[GEST.VOPEN] = dist(vopen, i); gestChance[GEST.HOPEN] = dist(hopen, i); gestChance[GEST.FINGERS] = dist(fingers, i); gestChance[GEST.SCISSORS] = dist(scissors, i); //list fold - get key of minimal value KeyValuePair<GEST,double> elem = gestChance.Aggregate((l, r) => l.Value < r.Value ? l : r); found[i] = (elem.Value < TOLERANCE) ? elem.Key : GEST.BLANK; if (elem.Key == GEST.FIST && (double)(X-x)/(Y-y) < .6) { found[i] = GEST.VOPEN; } gestureLabel[i] = labels[(int)found[i]]; } g1value.Text = gestureLabel[1]; g2value.Text = gestureLabel[0]; compactnessLbl.Text = observed[0, 1].ToString(format); blairLbl.Text = observed[1, 1].ToString(format); malinowskaLbl.Text = observed[2, 1].ToString(format); malzmodLbl.Text = observed[3, 1].ToString(format); feretLbl.Text = observed[4, 1].ToString(format); comp2.Text = observed[0, 0].ToString(format); blair2.Text = observed[1, 0].ToString(format); mal2.Text = observed[2, 0].ToString(format); malz2.Text = observed[3, 0].ToString(format); feret2.Text = observed[4, 0].ToString(format); /* for blobs not detected */ for (; i < 2; ++i) { observed[0, i] = observed[1, i] = observed[2, i] = observed[3, i] = observed[4, i] = NOT_FOUND; } imageGray = new Image<Gray, Byte>(bmp); imageGray = imageGray.Erode((int)nudErode.Value); imageGray = imageGray.Dilate((int)nudDilate.Value); imageBox2.Image = imageGray; //Zmiana pozycji myszki od środka ciężkości lewej ręki if (centerOfGravityLHandX * centerOfGravityLHandY != 0 && !blockMouseControl) { double smoothness = (double)nudSmoothness.Value; double sensitivity = (double)nudSensitivity.Value; int newPositionX = screenWidth - (int)(centerOfGravityLHandX / (imageGray.Width * .2) * sensitivity * screenWidth); //- imageGray.Width*1/5 int newPositionY = (int)((centerOfGravityLHandY - imageGray.Height * .5) / (imageGray.Height * .25) * sensitivity * screenHeight); int diffX = Cursor.Position.X + newPositionX; int diffY = Cursor.Position.Y - newPositionY; newPositionX = Cursor.Position.X - (int)(diffX / smoothness); newPositionY = Cursor.Position.Y - (int)(diffY / smoothness); MouseSimulating.SetMousePosition(newPositionX, newPositionY); //Wyliczanie akcji do podjęcia if (found[1] == GEST.BLANK || prevGestureLeft != found[1]) { frameCounterLeft = 0; prevGestureLeft = found[1]; } if (found[0] == GEST.BLANK || prevGestureRight != found[0]) { frameCounterRight = 0; prevGestureRight = found[0]; } if (frameCounterLeft == 30) //ile klatek musi - 30 kl/s { if (prevGestureLeft == GEST.FIST) MouseSimulating.PressLPM(); else if (prevGestureLeft == GEST.VOPEN) MouseSimulating.ReleaseLPM(); frameCounterLeft = 0; } else frameCounterLeft++; if (frameCounterRight == 30) { if (prevGestureRight == GEST.FIST) MouseSimulating.ClickLPM(); else if (prevGestureRight == GEST.SLAYER) MouseSimulating.ScrollUP(200); else if (prevGestureRight == GEST.VICTORY) MouseSimulating.ScrollDOWN(200); else if (prevGestureRight == GEST.FINGERS) MouseSimulating.ClickPPM(); else if (prevGestureRight == GEST.THUMBUP) KeyboardSimulating.SendCtrlC(); else if (prevGestureRight == GEST.THUMBLEFT) KeyboardSimulating.SendCtrlV(); else if (prevGestureRight == GEST.SCISSORS) KeyboardSimulating.SendCtrlX(); else if (prevGestureRight == GEST.HOPEN) { MouseSimulating.ClickLPM(); MouseSimulating.ClickLPM(); } else if (prevGestureRight == GEST.SHAKA) MouseSimulating.ClickMouseButton4(); frameCounterRight = 0; } else frameCounterRight++; } }
/* Odświezanie okna z obrazem */ void RefreshWindow(object sender, EventArgs arg) { //Pobieranie ramki image = capture.QueryFrame(); image = image.Flip(FLIP.HORIZONTAL); imageBox1.Image = image; //YCbCr or Bgr(RGB) //Warto zwrócić uwagę na to że Ycc to Y,Cr,Cb a nie Y,Cb,Cr, oraz Bgr to Blue,Green,Red if (radioButton1.Checked) { imageGray = image.Resize((double)nupScale.Value, INTER.CV_INTER_CUBIC).Convert <Ycc, Byte>(). InRange(new Ycc((double)nudW1.Value, (double)nudW3.Value, (double)nudW2.Value), new Ycc((double)nudW4.Value, (double)nudW6.Value, (double)nudW5.Value)); } else { imageGray = image.InRange(new Bgr((double)nudW3.Value, (double)nudW2.Value, (double)nudW1.Value), new Bgr((double)nudW6.Value, (double)nudW5.Value, (double)nudW4.Value)); } if (medianCB.Checked) { imageGray = imageGray.SmoothMedian((int)nudMedian.Value); } //Image<Gray, Byte> sgm = new Image<Gray, Byte>(imageGray.Size); Bitmap bmp = imageGray.ToBitmap(); bc.ProcessImage(bmp); Blob[] blob = bc.GetObjectsInformation(); //one hand version //int iters = bc.ObjectsCount > 1 ? 1 : bc.ObjectsCount; int iters = bc.ObjectsCount > 2 ? 2 : bc.ObjectsCount; if (iters > 1) { //both hands version //lewa reka to ta z prawej strony obrazu (zwierciadlo), nie zakladamy ze user gestykuluje na krzyz, keep it simple blob = blob.Take(2).OrderByDescending(a => a.CenterOfGravity.X).ToArray <Blob>(); } int centerOfGravityLHandX = 0, centerOfGravityLHandY = 0, centerOfGravityRHandX = 0, centerOfGravityRHandY = 0; string[] gestureLabel = new string[2]; int i = 0; for (; i < iters; ++i) { IntPoint minXY, maxXY; PointsCloud.GetBoundingRectangle(bc.GetBlobsEdgePoints(blob[i]), out minXY, out maxXY); Bitmap clonimage = (Bitmap)bmp.Clone(); BitmapData data = bmp.LockBits(new Rectangle(0, 0, imageGray.Width, imageGray.Height), ImageLockMode.ReadWrite, bmp.PixelFormat); Drawing.Rectangle(data, blob[i].Rectangle, Color.White); bmp.UnlockBits(data); int X = maxXY.X, Y = maxXY.Y; int x = minXY.X, y = minXY.Y; observed[0, i] = blob[i].Fullness; /* malinowska kryjaka liczy obwod ze wzoru na prostokąt, nasza liczy piksele krawedziowe */ //Malinowska(i) = (2*bb(3)+2*bb(4))/(2*sqrt(pi*S)) - 1; observed[2, i] = (double)(bc.GetBlobsEdgePoints(blob[i]).Count) / 2 / Math.Sqrt(Math.PI * blob[i].Area) - 1; //MalinowskaZ(i) = 2*sqrt(pi*S)/(2*bb(3)+2*bb(4)); observed[3, i] = 2 * Math.Sqrt(Math.PI * blob[i].Area) / (double)(bc.GetBlobsEdgePoints(blob[i]).Count); int gx = (int)blob[i].CenterOfGravity.X, gy = (int)blob[i].CenterOfGravity.Y; //Sprawdzenie która ręka prawa, a która lewa if (gx > centerOfGravityRHandX) { centerOfGravityLHandX = centerOfGravityRHandX; centerOfGravityLHandY = centerOfGravityRHandY; centerOfGravityRHandX = gx; centerOfGravityRHandY = gy; } else { centerOfGravityLHandX = gx; centerOfGravityLHandY = gy; } double blairsum = 0; int ftx = 0, ftxMax = 0; byte[, ,] dd = imageGray.Data; for (int j = y; j < Y; ++j) { if (ftx > ftxMax) { ftxMax = ftx; } ftx = 0;//bo moze sie zdazyc ze zliczy wiecej linii naraz, patrz: idealny prostokat for (int k = x; k < X; ++k) { if (dd[j, k, 0] != 0) { ++ftx; blairsum += (k - gx) * (k - gx) + (j - gy) * (j - gy);//distance squared } else { if (ftx > ftxMax) { ftxMax = ftx; } ftx = 0; } dd[j, k, 0] = 255; } } /* aby policzyc ftyMax trzeba puscic jeszcze jedna petle tak aby wewnetrzna szla po y-kach * ale mozna tez aproksymowac ftYmax przez boundingbox.Y, wtedy * przewidywalem najwieksze rozbieznosci przy skosnych lub dziurawych obiektach; * ale blad byl ponizej procenta, wiec szkoda tracic czas na kolejne O(n*n) * * int fty = 0, ftyMax = 0; * for (int j = x; j < X; ++j) { * if (fty > ftyMax) ftyMax = fty; * fty = 0; * for (int i = y; i < Y; ++i) * if (dd[i, j, 0] != 0) ++fty; * else { * if (fty > ftyMax) ftyMax = fty; * fty = 0; * } * } * feret = (double)ftxMax / ftyMax; */ observed[4, i] = (double)ftxMax / (Y - y); //feret observed[1, i] = (double)(blob[i].Area) / Math.Sqrt(2 * Math.PI * blairsum); //blair gestChance[GEST.SLAYER] = dist(slayer, i); gestChance[GEST.THUMBLEFT] = dist(thumbleft, i); gestChance[GEST.THUMBUP] = dist(thumbup, i); gestChance[GEST.SHAKA] = dist(shaka, i); gestChance[GEST.FIST] = dist(fist, i); gestChance[GEST.VICTORY] = dist(victory, i); gestChance[GEST.VOPEN] = dist(vopen, i); gestChance[GEST.HOPEN] = dist(hopen, i); gestChance[GEST.FINGERS] = dist(fingers, i); gestChance[GEST.SCISSORS] = dist(scissors, i); //list fold - get key of minimal value KeyValuePair <GEST, double> elem = gestChance.Aggregate((l, r) => l.Value < r.Value ? l : r); found[i] = (elem.Value < TOLERANCE) ? elem.Key : GEST.BLANK; if (elem.Key == GEST.FIST && (double)(X - x) / (Y - y) < .6) { found[i] = GEST.VOPEN; } gestureLabel[i] = labels[(int)found[i]]; } g1value.Text = gestureLabel[1]; g2value.Text = gestureLabel[0]; compactnessLbl.Text = observed[0, 1].ToString(format); blairLbl.Text = observed[1, 1].ToString(format); malinowskaLbl.Text = observed[2, 1].ToString(format); malzmodLbl.Text = observed[3, 1].ToString(format); feretLbl.Text = observed[4, 1].ToString(format); comp2.Text = observed[0, 0].ToString(format); blair2.Text = observed[1, 0].ToString(format); mal2.Text = observed[2, 0].ToString(format); malz2.Text = observed[3, 0].ToString(format); feret2.Text = observed[4, 0].ToString(format); /* for blobs not detected */ for (; i < 2; ++i) { observed[0, i] = observed[1, i] = observed[2, i] = observed[3, i] = observed[4, i] = NOT_FOUND; } imageGray = new Image <Gray, Byte>(bmp); imageGray = imageGray.Erode((int)nudErode.Value); imageGray = imageGray.Dilate((int)nudDilate.Value); imageBox2.Image = imageGray; //Zmiana pozycji myszki od środka ciężkości lewej ręki if (centerOfGravityLHandX * centerOfGravityLHandY != 0 && !blockMouseControl) { double smoothness = (double)nudSmoothness.Value; double sensitivity = (double)nudSensitivity.Value; int newPositionX = screenWidth - (int)(centerOfGravityLHandX / (imageGray.Width * .2) * sensitivity * screenWidth); //- imageGray.Width*1/5 int newPositionY = (int)((centerOfGravityLHandY - imageGray.Height * .5) / (imageGray.Height * .25) * sensitivity * screenHeight); int diffX = Cursor.Position.X + newPositionX; int diffY = Cursor.Position.Y - newPositionY; newPositionX = Cursor.Position.X - (int)(diffX / smoothness); newPositionY = Cursor.Position.Y - (int)(diffY / smoothness); MouseSimulating.SetMousePosition(newPositionX, newPositionY); //Wyliczanie akcji do podjęcia if (found[1] == GEST.BLANK || prevGestureLeft != found[1]) { frameCounterLeft = 0; prevGestureLeft = found[1]; } if (found[0] == GEST.BLANK || prevGestureRight != found[0]) { frameCounterRight = 0; prevGestureRight = found[0]; } if (frameCounterLeft == 30) //ile klatek musi - 30 kl/s { if (prevGestureLeft == GEST.FIST) { MouseSimulating.PressLPM(); } else if (prevGestureLeft == GEST.VOPEN) { MouseSimulating.ReleaseLPM(); } frameCounterLeft = 0; } else { frameCounterLeft++; } if (frameCounterRight == 30) { if (prevGestureRight == GEST.FIST) { MouseSimulating.ClickLPM(); } else if (prevGestureRight == GEST.SLAYER) { MouseSimulating.ScrollUP(200); } else if (prevGestureRight == GEST.VICTORY) { MouseSimulating.ScrollDOWN(200); } else if (prevGestureRight == GEST.FINGERS) { MouseSimulating.ClickPPM(); } else if (prevGestureRight == GEST.THUMBUP) { KeyboardSimulating.SendCtrlC(); } else if (prevGestureRight == GEST.THUMBLEFT) { KeyboardSimulating.SendCtrlV(); } else if (prevGestureRight == GEST.SCISSORS) { KeyboardSimulating.SendCtrlX(); } else if (prevGestureRight == GEST.HOPEN) { MouseSimulating.ClickLPM(); MouseSimulating.ClickLPM(); } else if (prevGestureRight == GEST.SHAKA) { MouseSimulating.ClickMouseButton4(); } frameCounterRight = 0; } else { frameCounterRight++; } } }