/* Odświezanie okna z obrazem */
void RefreshWindow(object sender, EventArgs arg)
{
//Pobieranie ramki
image = capture.QueryFrame();
image = image.Flip(FLIP.HORIZONTAL);
imageBox1.Image = image;
//YCbCr or Bgr(RGB)
//Warto zwrócić uwagę na to że Ycc to Y,Cr,Cb a nie Y,Cb,Cr, oraz Bgr to Blue,Green,Red
if (radioButton1.Checked)
imageGray = image.Resize((double)nupScale.Value, INTER.CV_INTER_CUBIC).Convert<Ycc, Byte>().
InRange(new Ycc((double)nudW1.Value, (double)nudW3.Value, (double)nudW2.Value), new Ycc((double)nudW4.Value, (double)nudW6.Value, (double)nudW5.Value));
else
imageGray = image.InRange(new Bgr((double)nudW3.Value, (double)nudW2.Value, (double)nudW1.Value), new Bgr((double)nudW6.Value, (double)nudW5.Value, (double)nudW4.Value));
if (medianCB.Checked)
imageGray = imageGray.SmoothMedian((int)nudMedian.Value);
//Image<Gray, Byte> sgm = new Image<Gray, Byte>(imageGray.Size);
Bitmap bmp = imageGray.ToBitmap();
bc.ProcessImage(bmp);
Blob[] blob = bc.GetObjectsInformation();
//one hand version
//int iters = bc.ObjectsCount > 1 ? 1 : bc.ObjectsCount;
int iters = bc.ObjectsCount > 2 ? 2 : bc.ObjectsCount;
if(iters > 1) {
//both hands version
//lewa reka to ta z prawej strony obrazu (zwierciadlo), nie zakladamy ze user gestykuluje na krzyz, keep it simple
blob = blob.Take(2).OrderByDescending(a => a.CenterOfGravity.X).ToArray<Blob>();
}
int centerOfGravityLHandX = 0, centerOfGravityLHandY = 0, centerOfGravityRHandX = 0, centerOfGravityRHandY = 0;
string[] gestureLabel = new string[2];
int i = 0;
for (; i < iters; ++i) {
IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(bc.GetBlobsEdgePoints(blob[i]), out minXY, out maxXY);
Bitmap clonimage = (Bitmap)bmp.Clone();
BitmapData data = bmp.LockBits(new Rectangle(0, 0, imageGray.Width, imageGray.Height), ImageLockMode.ReadWrite, bmp.PixelFormat);
Drawing.Rectangle(data, blob[i].Rectangle, Color.White);
bmp.UnlockBits(data);
int X = maxXY.X, Y = maxXY.Y;
int x = minXY.X, y = minXY.Y;
observed[0,i] = blob[i].Fullness;
/* malinowska kryjaka liczy obwod ze wzoru na prostokąt, nasza liczy piksele krawedziowe */
//Malinowska(i) = (2*bb(3)+2*bb(4))/(2*sqrt(pi*S)) - 1;
observed[2,i] = (double)(bc.GetBlobsEdgePoints(blob[i]).Count) / 2 / Math.Sqrt(Math.PI * blob[i].Area) - 1;
//MalinowskaZ(i) = 2*sqrt(pi*S)/(2*bb(3)+2*bb(4));
observed[3,i] = 2 * Math.Sqrt(Math.PI * blob[i].Area) / (double)(bc.GetBlobsEdgePoints(blob[i]).Count);
int gx = (int)blob[i].CenterOfGravity.X, gy = (int)blob[i].CenterOfGravity.Y;
//Sprawdzenie która ręka prawa, a która lewa
if (gx > centerOfGravityRHandX)
{
centerOfGravityLHandX = centerOfGravityRHandX;
centerOfGravityLHandY = centerOfGravityRHandY;
centerOfGravityRHandX = gx;
centerOfGravityRHandY = gy;
}
else
{
centerOfGravityLHandX = gx;
centerOfGravityLHandY = gy;
}
double blairsum = 0;
int ftx = 0, ftxMax = 0;
byte[, ,] dd = imageGray.Data;
for (int j = y; j < Y; ++j) {
if (ftx > ftxMax) ftxMax = ftx;
ftx = 0;//bo moze sie zdazyc ze zliczy wiecej linii naraz, patrz: idealny prostokat
for (int k = x; k < X; ++k) {
if (dd[j, k, 0] != 0) {
++ftx;
blairsum += (k - gx) * (k - gx) + (j - gy) * (j - gy);//distance squared
} else {
if (ftx > ftxMax) ftxMax = ftx;
ftx = 0;
}
dd[j, k, 0] = 255;
}
}
/* aby policzyc ftyMax trzeba puscic jeszcze jedna petle tak aby wewnetrzna szla po y-kach
* ale mozna tez aproksymowac ftYmax przez boundingbox.Y, wtedy
* przewidywalem najwieksze rozbieznosci przy skosnych lub dziurawych obiektach;
* ale blad byl ponizej procenta, wiec szkoda tracic czas na kolejne O(n*n)
int fty = 0, ftyMax = 0;
for (int j = x; j < X; ++j) {
if (fty > ftyMax) ftyMax = fty;
fty = 0;
for (int i = y; i < Y; ++i)
if (dd[i, j, 0] != 0)
++fty;
else {
if (fty > ftyMax) ftyMax = fty;
fty = 0;
}
}
feret = (double)ftxMax / ftyMax; */
observed[4,i] = (double)ftxMax / (Y-y);//feret
observed[1,i] = (double)(blob[i].Area) / Math.Sqrt(2 * Math.PI * blairsum);//blair
gestChance[GEST.SLAYER] = dist(slayer, i);
gestChance[GEST.THUMBLEFT] = dist(thumbleft, i);
gestChance[GEST.THUMBUP] = dist(thumbup, i);
gestChance[GEST.SHAKA] = dist(shaka, i);
gestChance[GEST.FIST] = dist(fist, i);
gestChance[GEST.VICTORY] = dist(victory, i);
gestChance[GEST.VOPEN] = dist(vopen, i);
gestChance[GEST.HOPEN] = dist(hopen, i);
gestChance[GEST.FINGERS] = dist(fingers, i);
gestChance[GEST.SCISSORS] = dist(scissors, i);
//list fold - get key of minimal value
KeyValuePair<GEST,double> elem = gestChance.Aggregate((l, r) => l.Value < r.Value ? l : r);
found[i] = (elem.Value < TOLERANCE) ? elem.Key : GEST.BLANK;
if (elem.Key == GEST.FIST && (double)(X-x)/(Y-y) < .6) {
found[i] = GEST.VOPEN;
}
gestureLabel[i] = labels[(int)found[i]];
}
g1value.Text = gestureLabel[1];
g2value.Text = gestureLabel[0];
compactnessLbl.Text = observed[0, 1].ToString(format);
blairLbl.Text = observed[1, 1].ToString(format);
malinowskaLbl.Text = observed[2, 1].ToString(format);
malzmodLbl.Text = observed[3, 1].ToString(format);
feretLbl.Text = observed[4, 1].ToString(format);
comp2.Text = observed[0, 0].ToString(format);
blair2.Text = observed[1, 0].ToString(format);
mal2.Text = observed[2, 0].ToString(format);
malz2.Text = observed[3, 0].ToString(format);
feret2.Text = observed[4, 0].ToString(format);
/* for blobs not detected */
for (; i < 2; ++i) {
observed[0, i] = observed[1, i] = observed[2, i] = observed[3, i] = observed[4, i] = NOT_FOUND;
}
imageGray = new Image<Gray, Byte>(bmp);
imageGray = imageGray.Erode((int)nudErode.Value);
imageGray = imageGray.Dilate((int)nudDilate.Value);
imageBox2.Image = imageGray;
//Zmiana pozycji myszki od środka ciężkości lewej ręki
if (centerOfGravityLHandX * centerOfGravityLHandY != 0 && !blockMouseControl)
{
double smoothness = (double)nudSmoothness.Value;
double sensitivity = (double)nudSensitivity.Value;
int newPositionX = screenWidth - (int)(centerOfGravityLHandX / (imageGray.Width * .2) * sensitivity * screenWidth); //- imageGray.Width*1/5
int newPositionY = (int)((centerOfGravityLHandY - imageGray.Height * .5) / (imageGray.Height * .25) * sensitivity * screenHeight);
int diffX = Cursor.Position.X + newPositionX;
int diffY = Cursor.Position.Y - newPositionY;
newPositionX = Cursor.Position.X - (int)(diffX / smoothness);
newPositionY = Cursor.Position.Y - (int)(diffY / smoothness);
MouseSimulating.SetMousePosition(newPositionX, newPositionY);
//Wyliczanie akcji do podjęcia
if (found[1] == GEST.BLANK || prevGestureLeft != found[1]) {
frameCounterLeft = 0;
prevGestureLeft = found[1];
}
if (found[0] == GEST.BLANK || prevGestureRight != found[0]) {
frameCounterRight = 0;
prevGestureRight = found[0];
}
if (frameCounterLeft == 30) //ile klatek musi - 30 kl/s
{
if (prevGestureLeft == GEST.FIST) MouseSimulating.PressLPM();
else if (prevGestureLeft == GEST.VOPEN) MouseSimulating.ReleaseLPM();
frameCounterLeft = 0;
} else frameCounterLeft++;
if (frameCounterRight == 30) {
if (prevGestureRight == GEST.FIST) MouseSimulating.ClickLPM();
else if (prevGestureRight == GEST.SLAYER) MouseSimulating.ScrollUP(200);
else if (prevGestureRight == GEST.VICTORY) MouseSimulating.ScrollDOWN(200);
else if (prevGestureRight == GEST.FINGERS) MouseSimulating.ClickPPM();
else if (prevGestureRight == GEST.THUMBUP) KeyboardSimulating.SendCtrlC();
else if (prevGestureRight == GEST.THUMBLEFT) KeyboardSimulating.SendCtrlV();
else if (prevGestureRight == GEST.SCISSORS) KeyboardSimulating.SendCtrlX();
else if (prevGestureRight == GEST.HOPEN) { MouseSimulating.ClickLPM(); MouseSimulating.ClickLPM(); }
else if (prevGestureRight == GEST.SHAKA) MouseSimulating.ClickMouseButton4();
frameCounterRight = 0;
}
else frameCounterRight++;
}
}
/* Odświezanie okna z obrazem */
void RefreshWindow(object sender, EventArgs arg)
{
//Pobieranie ramki
image = capture.QueryFrame();
image = image.Flip(FLIP.HORIZONTAL);
imageBox1.Image = image;
//YCbCr or Bgr(RGB)
//Warto zwrócić uwagę na to że Ycc to Y,Cr,Cb a nie Y,Cb,Cr, oraz Bgr to Blue,Green,Red
if (radioButton1.Checked)
{
imageGray = image.Resize((double)nupScale.Value, INTER.CV_INTER_CUBIC).Convert <Ycc, Byte>().
InRange(new Ycc((double)nudW1.Value, (double)nudW3.Value, (double)nudW2.Value), new Ycc((double)nudW4.Value, (double)nudW6.Value, (double)nudW5.Value));
}
else
{
imageGray = image.InRange(new Bgr((double)nudW3.Value, (double)nudW2.Value, (double)nudW1.Value), new Bgr((double)nudW6.Value, (double)nudW5.Value, (double)nudW4.Value));
}
if (medianCB.Checked)
{
imageGray = imageGray.SmoothMedian((int)nudMedian.Value);
}
//Image<Gray, Byte> sgm = new Image<Gray, Byte>(imageGray.Size);
Bitmap bmp = imageGray.ToBitmap();
bc.ProcessImage(bmp);
Blob[] blob = bc.GetObjectsInformation();
//one hand version
//int iters = bc.ObjectsCount > 1 ? 1 : bc.ObjectsCount;
int iters = bc.ObjectsCount > 2 ? 2 : bc.ObjectsCount;
if (iters > 1)
{
//both hands version
//lewa reka to ta z prawej strony obrazu (zwierciadlo), nie zakladamy ze user gestykuluje na krzyz, keep it simple
blob = blob.Take(2).OrderByDescending(a => a.CenterOfGravity.X).ToArray <Blob>();
}
int centerOfGravityLHandX = 0, centerOfGravityLHandY = 0, centerOfGravityRHandX = 0, centerOfGravityRHandY = 0;
string[] gestureLabel = new string[2];
int i = 0;
for (; i < iters; ++i)
{
IntPoint minXY, maxXY;
PointsCloud.GetBoundingRectangle(bc.GetBlobsEdgePoints(blob[i]), out minXY, out maxXY);
Bitmap clonimage = (Bitmap)bmp.Clone();
BitmapData data = bmp.LockBits(new Rectangle(0, 0, imageGray.Width, imageGray.Height), ImageLockMode.ReadWrite, bmp.PixelFormat);
Drawing.Rectangle(data, blob[i].Rectangle, Color.White);
bmp.UnlockBits(data);
int X = maxXY.X, Y = maxXY.Y;
int x = minXY.X, y = minXY.Y;
observed[0, i] = blob[i].Fullness;
/* malinowska kryjaka liczy obwod ze wzoru na prostokąt, nasza liczy piksele krawedziowe */
//Malinowska(i) = (2*bb(3)+2*bb(4))/(2*sqrt(pi*S)) - 1;
observed[2, i] = (double)(bc.GetBlobsEdgePoints(blob[i]).Count) / 2 / Math.Sqrt(Math.PI * blob[i].Area) - 1;
//MalinowskaZ(i) = 2*sqrt(pi*S)/(2*bb(3)+2*bb(4));
observed[3, i] = 2 * Math.Sqrt(Math.PI * blob[i].Area) / (double)(bc.GetBlobsEdgePoints(blob[i]).Count);
int gx = (int)blob[i].CenterOfGravity.X, gy = (int)blob[i].CenterOfGravity.Y;
//Sprawdzenie która ręka prawa, a która lewa
if (gx > centerOfGravityRHandX)
{
centerOfGravityLHandX = centerOfGravityRHandX;
centerOfGravityLHandY = centerOfGravityRHandY;
centerOfGravityRHandX = gx;
centerOfGravityRHandY = gy;
}
else
{
centerOfGravityLHandX = gx;
centerOfGravityLHandY = gy;
}
double blairsum = 0;
int ftx = 0, ftxMax = 0;
byte[, ,] dd = imageGray.Data;
for (int j = y; j < Y; ++j)
{
if (ftx > ftxMax)
{
ftxMax = ftx;
}
ftx = 0;//bo moze sie zdazyc ze zliczy wiecej linii naraz, patrz: idealny prostokat
for (int k = x; k < X; ++k)
{
if (dd[j, k, 0] != 0)
{
++ftx;
blairsum += (k - gx) * (k - gx) + (j - gy) * (j - gy);//distance squared
}
else
{
if (ftx > ftxMax)
{
ftxMax = ftx;
}
ftx = 0;
}
dd[j, k, 0] = 255;
}
}
/* aby policzyc ftyMax trzeba puscic jeszcze jedna petle tak aby wewnetrzna szla po y-kach
* ale mozna tez aproksymowac ftYmax przez boundingbox.Y, wtedy
* przewidywalem najwieksze rozbieznosci przy skosnych lub dziurawych obiektach;
* ale blad byl ponizej procenta, wiec szkoda tracic czas na kolejne O(n*n)
*
* int fty = 0, ftyMax = 0;
* for (int j = x; j < X; ++j) {
* if (fty > ftyMax) ftyMax = fty;
* fty = 0;
* for (int i = y; i < Y; ++i)
* if (dd[i, j, 0] != 0)
++fty;
* else {
* if (fty > ftyMax) ftyMax = fty;
* fty = 0;
* }
* }
* feret = (double)ftxMax / ftyMax; */
observed[4, i] = (double)ftxMax / (Y - y); //feret
observed[1, i] = (double)(blob[i].Area) / Math.Sqrt(2 * Math.PI * blairsum); //blair
gestChance[GEST.SLAYER] = dist(slayer, i);
gestChance[GEST.THUMBLEFT] = dist(thumbleft, i);
gestChance[GEST.THUMBUP] = dist(thumbup, i);
gestChance[GEST.SHAKA] = dist(shaka, i);
gestChance[GEST.FIST] = dist(fist, i);
gestChance[GEST.VICTORY] = dist(victory, i);
gestChance[GEST.VOPEN] = dist(vopen, i);
gestChance[GEST.HOPEN] = dist(hopen, i);
gestChance[GEST.FINGERS] = dist(fingers, i);
gestChance[GEST.SCISSORS] = dist(scissors, i);
//list fold - get key of minimal value
KeyValuePair <GEST, double> elem = gestChance.Aggregate((l, r) => l.Value < r.Value ? l : r);
found[i] = (elem.Value < TOLERANCE) ? elem.Key : GEST.BLANK;
if (elem.Key == GEST.FIST && (double)(X - x) / (Y - y) < .6)
{
found[i] = GEST.VOPEN;
}
gestureLabel[i] = labels[(int)found[i]];
}
g1value.Text = gestureLabel[1];
g2value.Text = gestureLabel[0];
compactnessLbl.Text = observed[0, 1].ToString(format);
blairLbl.Text = observed[1, 1].ToString(format);
malinowskaLbl.Text = observed[2, 1].ToString(format);
malzmodLbl.Text = observed[3, 1].ToString(format);
feretLbl.Text = observed[4, 1].ToString(format);
comp2.Text = observed[0, 0].ToString(format);
blair2.Text = observed[1, 0].ToString(format);
mal2.Text = observed[2, 0].ToString(format);
malz2.Text = observed[3, 0].ToString(format);
feret2.Text = observed[4, 0].ToString(format);
/* for blobs not detected */
for (; i < 2; ++i)
{
observed[0, i] = observed[1, i] = observed[2, i] = observed[3, i] = observed[4, i] = NOT_FOUND;
}
imageGray = new Image <Gray, Byte>(bmp);
imageGray = imageGray.Erode((int)nudErode.Value);
imageGray = imageGray.Dilate((int)nudDilate.Value);
imageBox2.Image = imageGray;
//Zmiana pozycji myszki od środka ciężkości lewej ręki
if (centerOfGravityLHandX * centerOfGravityLHandY != 0 && !blockMouseControl)
{
double smoothness = (double)nudSmoothness.Value;
double sensitivity = (double)nudSensitivity.Value;
int newPositionX = screenWidth - (int)(centerOfGravityLHandX / (imageGray.Width * .2) * sensitivity * screenWidth); //- imageGray.Width*1/5
int newPositionY = (int)((centerOfGravityLHandY - imageGray.Height * .5) / (imageGray.Height * .25) * sensitivity * screenHeight);
int diffX = Cursor.Position.X + newPositionX;
int diffY = Cursor.Position.Y - newPositionY;
newPositionX = Cursor.Position.X - (int)(diffX / smoothness);
newPositionY = Cursor.Position.Y - (int)(diffY / smoothness);
MouseSimulating.SetMousePosition(newPositionX, newPositionY);
//Wyliczanie akcji do podjęcia
if (found[1] == GEST.BLANK || prevGestureLeft != found[1])
{
frameCounterLeft = 0;
prevGestureLeft = found[1];
}
if (found[0] == GEST.BLANK || prevGestureRight != found[0])
{
frameCounterRight = 0;
prevGestureRight = found[0];
}
if (frameCounterLeft == 30) //ile klatek musi - 30 kl/s
{
if (prevGestureLeft == GEST.FIST)
{
MouseSimulating.PressLPM();
}
else if (prevGestureLeft == GEST.VOPEN)
{
MouseSimulating.ReleaseLPM();
}
frameCounterLeft = 0;
}
else
{
frameCounterLeft++;
}
if (frameCounterRight == 30)
{
if (prevGestureRight == GEST.FIST)
{
MouseSimulating.ClickLPM();
}
else if (prevGestureRight == GEST.SLAYER)
{
MouseSimulating.ScrollUP(200);
}
else if (prevGestureRight == GEST.VICTORY)
{
MouseSimulating.ScrollDOWN(200);
}
else if (prevGestureRight == GEST.FINGERS)
{
MouseSimulating.ClickPPM();
}
else if (prevGestureRight == GEST.THUMBUP)
{
KeyboardSimulating.SendCtrlC();
}
else if (prevGestureRight == GEST.THUMBLEFT)
{
KeyboardSimulating.SendCtrlV();
}
else if (prevGestureRight == GEST.SCISSORS)
{
KeyboardSimulating.SendCtrlX();
}
else if (prevGestureRight == GEST.HOPEN)
{
MouseSimulating.ClickLPM(); MouseSimulating.ClickLPM();
}
else if (prevGestureRight == GEST.SHAKA)
{
MouseSimulating.ClickMouseButton4();
}
frameCounterRight = 0;
}
else
{
frameCounterRight++;
}
}
}
