public bool ProcessFrame()
{
if (CurrentFrame >= TotalFrames - 1)
{
return(false);
}
if (_CCapture != null)
{
var r = _CCapture.Grab();
var imgc = _CCapture.RetrieveBgrFrame();
ImageConverter.UpdateWriteBMP(colorWriteBitmap, imgc.ToBitmap());
}
if (_DCapture != null)
{
var r = _DCapture.Grab();
var img = _DCapture.RetrieveBgrFrame();
int framenumber = (int)_DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_FRAMES);
//Console.WriteLine("frame:"+framenumber.ToString());
CurrentFrame = framenumber - 1;
if (img == null)
{
return(false);
}
//Show time stamp
double time_index = _DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_MSEC);
PointF rightVector = new PointF(-10, -10);
PointF leftVector = new PointF(10, -10);
bool isSkip = false;
bool leftHandRaise = false;
if (CurrentFrame >= sktList.Count || !sktList[CurrentFrame].Tracked)
{
// no skeleton detected
headDepth = 0;
}
else
{
headPosition = sktList[CurrentFrame][MyJointType.Head].PosDepth;
headDepth = Math.Min((int)img[headPosition.Y, headPosition.X].Green, (int)img[headPosition.Y, headPosition.X].Red);
headDepth = Math.Min(headDepth, (int)img[headPosition.Y, headPosition.X].Blue);
PointF hr = sktList[CurrentFrame][MyJointType.HandR].PosDepth;
PointF hl = sktList[CurrentFrame][MyJointType.HandL].PosDepth;
PointF er = sktList[CurrentFrame][MyJointType.ElbowR].PosDepth;
PointF el = sktList[CurrentFrame][MyJointType.ElbowL].PosDepth;
PointF hip = sktList[CurrentFrame][MyJointType.HipCenter].PosDepth;
// hand is lower than hip
//Console.WriteLine(sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y);
//Console.WriteLine(sktList[CurrentFrame][MyJointType.ElbowR].Pos3D.Y);
//Console.WriteLine(sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y);
//Console.WriteLine("-------------");
if (sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y <
sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y + 0.05)
{
isSkip = true;
}
if (sktList[CurrentFrame][MyJointType.HandL].Pos3D.Y >
-0.02f)
{
leftHandRaise = true;
}
rightVector.X = (hr.X - er.X);
rightVector.Y = (hr.Y - er.Y);
leftVector.X = (hl.X - el.X);
leftVector.Y = (hl.Y - el.Y);
}
#region temp
//isSkip = false;
//leftHandRaise = false;
//rightVector = new PointF(-10, -10);
//leftVector = new PointF(10, -10);
//headDepth = (int)img[85, 315].Blue;
#endregion
// Console.WriteLine("headdepth："+headDepth.ToString());
//***********cull image*****************
double cull = headDepth - CullingThresh;
var depthImg = img.ThresholdToZeroInv(new Bgr(cull, cull, cull));
//Image<Gray, Byte> depthImg = img.Convert<Gray, byte>().ThresholdBinary(new Gray(160), new Gray(255));
var sw = Stopwatch.StartNew();
int handDepth = (int)(2600.0 / 255 * cull + 400);
HandShapeModel handModel = null;
Image <Gray, Byte> rightFront = null;
Image <Gray, Byte> leftFront = null;
// isskip is invalid coz no hip data
isSkip = false;
if (cull > 0)
{
handModel = m_opencv.FindHandPart(ref depthImg, out rightFront, out leftFront, handDepth, rightVector, leftVector, leftHandRaise);
}
viewer.Image = depthImg;
// no hands detected
if (handModel == null)
{
handModel = new HandShapeModel(0, HandEnum.None);
}
var row = new CsvRow();
row.Add(CurrentFrame.ToString());
row.Add(handModel.type.ToString());
switch (handModel.type)
{
case HandEnum.Intersect:
case HandEnum.Right:
row.Add(handModel.handPosRight.center.X.ToString());
row.Add(handModel.handPosRight.center.Y.ToString());
row.Add(handModel.handPosRight.size.Width.ToString());
row.Add(handModel.handPosRight.size.Height.ToString());
row.Add(handModel.handPosRight.angle.ToString());
break;
case HandEnum.Both:
row.Add(handModel.handPosRight.center.X.ToString());
row.Add(handModel.handPosRight.center.Y.ToString());
row.Add(handModel.handPosRight.size.Width.ToString());
row.Add(handModel.handPosRight.size.Height.ToString());
row.Add(handModel.handPosRight.angle.ToString());
row.Add(handModel.handPosLeft.center.X.ToString());
row.Add(handModel.handPosLeft.center.Y.ToString());
row.Add(handModel.handPosLeft.size.Width.ToString());
row.Add(handModel.handPosLeft.size.Height.ToString());
row.Add(handModel.handPosLeft.angle.ToString());
break;
}
row.Add(cull.ToString());
labelWriter.WriteRow(row);
if (rightFront != null)
{
ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, rightFront.ToBitmap());
}
ImageConverter.UpdateWriteBMP(depthWriteBitmap, img.ToBitmap());
if (CurrentFrame < TotalFrames - 1)
{
return(true);
}
}
return(false);
}
public bool ProcessFrame()
{
if (CurrentFrame >= TotalFrames -1)
{
return false;
}
if (_CCapture != null)
{
var r = _CCapture.Grab();
var imgc = _CCapture.RetrieveBgrFrame();
ImageConverter.UpdateWriteBMP(colorWriteBitmap, imgc.ToBitmap());
}
if (_DCapture != null)
{
var r = _DCapture.Grab();
var img = _DCapture.RetrieveBgrFrame();
int framenumber = (int)_DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_FRAMES);
//Console.WriteLine("frame:"+framenumber.ToString());
CurrentFrame = framenumber - 1;
if (img == null)
{
return false;
}
//Show time stamp
double time_index = _DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_MSEC);
PointF rightVector = new PointF(-10, -10);
PointF leftVector = new PointF(10, -10);
bool isSkip = false;
bool leftHandRaise = false;
if (CurrentFrame >= sktList.Count || !sktList[CurrentFrame].Tracked)
{
// no skeleton detected
headDepth = 0;
}
else
{
headPosition = sktList[CurrentFrame][MyJointType.Head].PosDepth;
headDepth = Math.Min((int)img[headPosition.Y, headPosition.X].Green,(int)img[headPosition.Y, headPosition.X].Red);
headDepth = Math.Min(headDepth, (int) img[headPosition.Y, headPosition.X].Blue);
PointF hr = sktList[CurrentFrame][MyJointType.HandR].PosDepth;
PointF hl = sktList[CurrentFrame][MyJointType.HandL].PosDepth;
PointF er = sktList[CurrentFrame][MyJointType.ElbowR].PosDepth;
PointF el = sktList[CurrentFrame][MyJointType.ElbowL].PosDepth;
PointF hip = sktList[CurrentFrame][MyJointType.HipCenter].PosDepth;
// hand is lower than hip
//Console.WriteLine(sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y);
//Console.WriteLine(sktList[CurrentFrame][MyJointType.ElbowR].Pos3D.Y);
//Console.WriteLine(sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y);
//Console.WriteLine("-------------");
if (sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y <
sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y + 0.05)
{
isSkip = true;
}
if (sktList[CurrentFrame][MyJointType.HandL].Pos3D.Y >
-0.02f)
{
leftHandRaise = true;
}
rightVector.X = (hr.X - er.X);
rightVector.Y = (hr.Y - er.Y);
leftVector.X = (hl.X - el.X);
leftVector.Y = (hl.Y - el.Y);
}
#region temp
//isSkip = false;
//leftHandRaise = false;
//rightVector = new PointF(-10, -10);
//leftVector = new PointF(10, -10);
//headDepth = (int)img[85, 315].Blue;
#endregion
// Console.WriteLine("headdepth："+headDepth.ToString());
//***********cull image*****************
double cull = headDepth - CullingThresh;
var depthImg = img.ThresholdToZeroInv(new Bgr(cull, cull, cull));
//Image<Gray, Byte> depthImg = img.Convert<Gray, byte>().ThresholdBinary(new Gray(160), new Gray(255));
var sw = Stopwatch.StartNew();
int handDepth = (int)(2600.0 / 255 * cull + 400);
HandShapeModel handModel = null;
Image<Gray, Byte> rightFront = null;
Image<Gray, Byte> leftFront = null;
// isskip is invalid coz no hip data
isSkip = false;
if (cull> 0)
{
handModel = m_opencv.FindHandPart(ref depthImg, out rightFront, out leftFront, handDepth, rightVector, leftVector, leftHandRaise);
}
viewer.Image = depthImg;
// no hands detected
if (handModel == null)
{
handModel = new HandShapeModel(0, HandEnum.None);
}
var row = new CsvRow();
row.Add(CurrentFrame.ToString());
row.Add(handModel.type.ToString());
switch (handModel.type)
{
case HandEnum.Intersect:
case HandEnum.Right:
row.Add(handModel.handPosRight.center.X.ToString());
row.Add(handModel.handPosRight.center.Y.ToString());
row.Add(handModel.handPosRight.size.Width.ToString());
row.Add(handModel.handPosRight.size.Height.ToString());
row.Add(handModel.handPosRight.angle.ToString());
break;
case HandEnum.Both:
row.Add(handModel.handPosRight.center.X.ToString());
row.Add(handModel.handPosRight.center.Y.ToString());
row.Add(handModel.handPosRight.size.Width.ToString());
row.Add(handModel.handPosRight.size.Height.ToString());
row.Add(handModel.handPosRight.angle.ToString());
row.Add(handModel.handPosLeft.center.X.ToString());
row.Add(handModel.handPosLeft.center.Y.ToString());
row.Add(handModel.handPosLeft.size.Width.ToString());
row.Add(handModel.handPosLeft.size.Height.ToString());
row.Add(handModel.handPosLeft.angle.ToString());
break;
}
row.Add(cull.ToString());
labelWriter.WriteRow(row);
if (rightFront != null)
{
ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, rightFront.ToBitmap());
}
ImageConverter.UpdateWriteBMP(depthWriteBitmap, img.ToBitmap());
if (CurrentFrame < TotalFrames -1)
{
return true;
}
}
return false;
}
public unsafe HandShapeModel FindHandPart(
ref Image<Bgr, byte> image,
out Image<Gray, Byte> rightFront,
out Image<Gray, Byte> leftFront,
int handDepth,
PointF rightVector,
PointF leftVector,
bool leftHandRaise
)
{
//process hand size
if (handDepth > 0)
{
begin = (int)(100.0 * 240.0 / handDepth / 0.39);
end = (int)(220 * 240.0 / handDepth / 0.39);
minLength = (int)(150 * 240.0 / handDepth / 0.39);
}
else
{
rightFront = null;
leftFront = null;
return null;
}
//Console.WriteLine(begin);
//Console.WriteLine(end);
//Console.WriteLine(minLength);
rightFront = null;
leftFront = null;
HandShapeModel model = null;
//find contour
grayImg = image.Convert<Gray, byte>();
Image<Gray, Byte> binaryImg = GetBinaryImg(image);
image = binaryImg.Convert<Bgr, byte>();
List<MCvBox2D> rectList = FindContourMBox(binaryImg);
//draw contour
foreach (var rect in rectList)
{
DrawPoly(rect.GetVertices().ToPoints(), image, new MCvScalar(255, 0, 0));
}
// find hand
rectList = rectList.OrderByDescending(x => x.GetTrueArea()).ToList();
MCvBox2D rightHand;
MCvBox2D leftHand;
Font textFont = new Font(FontFamily.Families[0], 20);
// count hands number
using (Graphics g = Graphics.FromImage(image.Bitmap))
{
// 3 conditions in total
if (rectList.Count() >= 2)//two hands
{
if (rectList[0].center.X > rectList[1].center.X)
{
rightHand = rectList[0];
leftHand = rectList[1];
}
else
{
rightHand = rectList[1];
leftHand = rectList[0];
}
// mark intersect state
Intersect = rightHand.MinAreaRect().IsCloseTo(leftHand.MinAreaRect(), 5);
//right hand
MCvBox2D SplittedRightHand = SplitHand(rightHand, HandEnum.Right, rightVector);
rightFront = GetSubImage<Gray>(binaryImg, SplittedRightHand, rightHand.angle);
float[] rightHog = CalHog(rightFront);
DrawHand(SplittedRightHand, image, HandEnum.Right);
//left hand
MCvBox2D SplittedLeftHand = SplitHand(leftHand, HandEnum.Left, leftVector);
leftFront = GetSubImage<Gray>(binaryImg, SplittedLeftHand, leftHand.angle);
float[] leftHog = CalHog(leftFront);
DrawHand(SplittedLeftHand, image, HandEnum.Left);
g.DrawString("left and right", textFont, Brushes.Red, 0, 20);
model = new HandShapeModel(hogSize, HandEnum.Both);
model.hogLeft = leftHog;
model.hogRight = rightHog;
model.handPosRight = SplittedRightHand;
model.handPosLeft = SplittedLeftHand;
}
else if (rectList.Count() == 1) // one rectangle
{
string text = "";
leftFront = null;
if (leftHandRaise || Intersect)
{
text = "Two hands";
var recs = FindContourRect(binaryImg);
if (recs.Count > 0)
{
var rec = recs[0];
rightFront = GetSubImageByRect<Gray>(binaryImg, rec);
MCvBox2D box = rec.ToCvBox2D();
DrawHand(box, image, HandEnum.Intersect);
float[] TwoHandHOG = CalHog(rightFront);
model = new HandShapeModel(hogSize, HandEnum.Intersect);
model.hogRight = TwoHandHOG;
model.handPosRight = box;
}
}
else
{
text = "right";
MCvBox2D SplittedRightHand = SplitHand(rectList[0], HandEnum.Right, rightVector);
//Console.WriteLine(SplittedRightHand.size.Height);
rightFront = GetSubImage<Gray>(binaryImg, SplittedRightHand, rectList[0].angle);
DrawHand(SplittedRightHand, image, HandEnum.Right);
float[] TwoHandHOG = CalHog(rightFront);
model = new HandShapeModel(hogSize, HandEnum.Right);
model.hogRight = TwoHandHOG;
model.handPosRight = SplittedRightHand;
}
g.DrawString(text, textFont, Brushes.Red, 0, 20);
}
else // no hand detected
{
Intersect = false;
}
}
return model;
}
public void WriteFrameData(HandShapeModel handModel)
{
if (connection == null || command == null) return;
// cmd = String.Format("INSERT INTO FrameData " +
// "(FrameDataId, SignInfoId, FrameCount, OffsetedFrameCount, " + //0, 1, 2
//"SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " +
//"SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " +
//"SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " +
//"SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " +
//"SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " +
//"SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " +
//"SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " +
//"SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " +
//"SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " +
//"SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " +
//"SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " +
//"SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " +
//"SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " +
//"SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " +
// "HandCount, " +
// "Hand0FingertipCount, " +
// "Hand0Fingertip0X, Hand0Fingertip0Y, Hand0Fingertip0Z, " +
// "Hand0Fingertip1X, Hand0Fingertip1Y, Hand0Fingertip1Z, " +
// "Hand0Fingertip2X, Hand0Fingertip2Y, Hand0Fingertip2Z, " +
// "Hand0Fingertip3X, Hand0Fingertip3Y, Hand0Fingertip3Z, " +
// "Hand0Fingertip4X, Hand0Fingertip4Y, Hand0Fingertip4Z, " +
// "Hand0EllipseCenterX, Hand0EllipseCenterY, " +
// "Hand0EllipseMajorAxis, Hand0EllipseMinorAxis, " +
// "Hand0EllipseAspectRatio, Hand0AxisTheta, " +
// "Hand1FingertipCount, " +
// "Hand1Fingertip0X, Hand1Fingertip0Y, Hand1Fingertip0Z, " +
// "Hand1Fingertip1X, Hand1Fingertip1Y, Hand1Fingertip1Z, " +
// "Hand1Fingertip2X, Hand1Fingertip2Y, Hand1Fingertip2Z, " +
// "Hand1Fingertip3X, Hand1Fingertip3Y, Hand1Fingertip3Z, " +
// "Hand1Fingertip4X, Hand1Fingertip4Y, Hand1Fingertip4Z, " +
// "Hand1EllipseCenterX, Hand1EllipseCenterY, " +
// "Hand1EllipseMajorAxis, Hand1EllipseMinorAxis, " +
// "Hand1EllipseAspectRatio, Hand1AxisTheta) " +
// "VALUES(NULL, {0}, {1}, {2}{3})",
// SignInfoId, frameData.frameCount, frameData.OffsetedFrameCount, frameData.GetFrameDataArgString()
// );
// insertions.Add(cmd);
//} else {
// if (insertions.Count > 0) { //Batch insertion
// var sw = Stopwatch.StartNew();
// reader.Close();
// SQLiteTransaction trans = connection.BeginTransaction(); // <-------------------
// try
// {
// foreach (string insertion in insertions)
// {
// command.CommandText = insertion;
// command.ExecuteNonQuery();
// }
// insertions.Clear();
// trans.Commit(); // <-------------------
// }
// catch
// {
// trans.Rollback(); // <-------------------
// throw; // <-------------------
// }
// Console.WriteLine("save data:" + sw.ElapsedMilliseconds);
// }
//}
//if (!reader.IsClosed) {
// reader.Close();
//}
////if (cmd != String.Empty) {
//// command.CommandText = cmd;
//// command.ExecuteNonQuery();
////}
}
private void AllFrameReady(object sender, AllFramesReadyEventArgs e)
{
//Console.Clear();
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame != null)
{
skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
skeletonFrame.CopySkeletonDataTo(this.skeletons);
Skeleton skel = skeletons[0];
if (skel.TrackingState == SkeletonTrackingState.Tracked)
{
SkeletonPoint rightHand = skeletons[0].Joints[JointType.HandRight].Position;
SkeletonPoint head = skeletons[0].Joints[JointType.Head].Position;
rightHandPosition = SkeletonPointToScreen(rightHand);
headPosition = SkeletonPointToScreen(head);
}
}
}
using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
{
if (colorFrame != null)
{
// Copy the pixel data from the image to a temporary array
colorFrame.CopyPixelDataTo(this.colorPixels);
// Write the pixel data into our bitmap
this.ColorWriteBitmap.WritePixels(
new System.Windows.Int32Rect(0, 0, this.ColorWriteBitmap.PixelWidth, this.ColorWriteBitmap.PixelHeight),
this.colorPixels,
this.ColorWriteBitmap.PixelWidth * sizeof(int),
0);
}
}
using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
{
if (depthFrame != null)
{
var sw = Stopwatch.StartNew();
// Copy the pixel data from the image to a temporary array
depthFrame.CopyDepthImagePixelDataTo(this.depthPixels);
// short[] depthData = new short[depthFrame.PixelDataLength];
// Get the min and max reliable depth for the current frame
int minDepth = depthFrame.MinDepth;
int maxDepth = depthFrame.MaxDepth;
int width = depthFrame.Width;
int height = depthFrame.Height;
if (headPosition.X == 0)
{
headDepth = 100;
}
else
{
headDepth = depthPixels[headPosition.X + headPosition.Y * 640].Depth;
}
sw.Restart();
//*********** Convert cull and transform*****************
colorizer.TransformCullAndConvertDepthFrame(
depthPixels, minDepth, maxDepth, colorPixels,
AngleRotateTan,
(short)(headDepth - (short)CullingThresh), headPosition);
Image <Bgra, byte> depthImg;
//Console.WriteLine("iteration:" + sw.ElapsedMilliseconds);
sw.Restart();
Image <Gray, Byte> rightFront = null;
Image <Gray, Byte> leftFront = null;
depthImg = ImageConverter.Array2Image(colorPixels, width, height, width * 4);
PointF rightVector = PointF.Empty;
PointF leftVector = PointF.Empty;
bool isSkip = false;
bool leftHandRaise = false;
if (skeletons != null && skeletons[0].TrackingState == SkeletonTrackingState.Tracked)
{
PointF hr = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandRight].Position);
PointF hl = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandLeft].Position);
PointF er = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowRight].Position);
PointF el = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowLeft].Position);
PointF hip = SkeletonPointToScreen(skeletons[0].Joints[JointType.HipCenter].Position);
// hand is lower than hip
//Console.WriteLine(skeletons[0].Joints[JointType.HandRight].Position.Y);
//Console.WriteLine(skeletons[0].Joints[JointType.HipCenter].Position.Y);
//Console.WriteLine("-------------");
if (skeletons[0].Joints[JointType.HandRight].Position.Y <
skeletons[0].Joints[JointType.HipCenter].Position.Y + 0.05)
{
isSkip = true;
}
if (skeletons[0].Joints[JointType.HandLeft].Position.Y >
skeletons[0].Joints[JointType.HipCenter].Position.Y)
{
leftHandRaise = true;
}
rightVector.X = (hr.X - er.X);
rightVector.Y = (hr.Y - er.Y);
leftVector.X = (hl.X - el.X);
leftVector.Y = (hl.Y - el.Y);
}
HandShapeModel handModel = null;
if (!isSkip)
{
//handModel = m_OpenCVController.FindHandPart(ref depthImg, out rightFront, out leftFront, headDepth - (int)CullingThresh, rightVector, leftVector,leftHandRaise);
}
// no hands detected
if (handModel == null)
{
handModel = new HandShapeModel(0, HandEnum.None);
}
//sw.Restart();
// database processing
DBManager db = DBManager.GetSingleton();
if (db != null)
{
if (skeletons != null)
{
handModel.SetSkeletonData(skeletons[0]);
}
db.AddFrameData(handModel);
}
// not recording show prob
else
{
//Image<Bgr, byte>[] result = HandShapeClassifier.GetSingleton()
//.RecognizeGesture(handModel.hogRight, 3);
////Console.WriteLine(sw.ElapsedMilliseconds);
//if (result != null)
//{
// ImageConverter.UpdateWriteBMP(WrtBMP_Candidate1, result[0].Convert<Gray, byte>().ToBitmap());
// ImageConverter.UpdateWriteBMP(WrtBMP_Candidate2, result[1].Convert<Gray, byte>().ToBitmap());
// ImageConverter.UpdateWriteBMP(WrtBMP_Candidate3, result[2].Convert<Gray, byte>().ToBitmap());
//}
}
string currentSign = db == null ? "0" : db.CurrentSign.ToString();
string path = @"J:\Kinect data\Aaron 141-180\hands\" + currentSign + " " + handModel.frame.ToString();
// UI update
if (rightFront != null)
{
Bitmap right = rightFront.ToBitmap();
//right.Save(path + " r.jpg");
ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, right);
}
if (leftFront != null)
{
Bitmap left = leftFront.ToBitmap();
//left.Save(path + " l.jpg");
ImageConverter.UpdateWriteBMP(WrtBMP_LeftHandFront, left);
}
if (sw.ElapsedMilliseconds > 15)
{
Console.WriteLine("Find hand:" + sw.ElapsedMilliseconds);
}
sw.Restart();
//**************************draw gray histogram
//Bitmap histo = m_OpenCVController.Histogram(depthImg);
//ImageConverter.UpdateWriteBMP(GrayWriteBitmap, histo);
// draw hand position from kinect
// DrawHandPosition(depthBMP, rightHandPosition, System.Drawing.Brushes.Purple);
//*******************upadte UI
ImageConverter.UpdateWriteBMP(DepthWriteBitmap, depthImg.ToBitmap());
// Console.WriteLine("Update UI:" + sw.ElapsedMilliseconds);
}
}
}
public bool AddFrameData(HandShapeModel hand)
{
currentFrame++;
if (!Begin || hand == null)
{
return false;
}
try
{
SQLiteCommand insertCommand = new SQLiteCommand(connection);
string insertframe =
String.Format("INSERT OR REPLACE INTO FrameData (" +
"SampleIndex,FrameNumber," +
"SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " +
"SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " +
"SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " +
"SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " +
"SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " +
"SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " +
"SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " +
"SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " +
"SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " +
"SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " +
"SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " +
"SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " +
"SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " +
"SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " +
"RightHandHOG, LeftHandHOG) VALUES" +
"(@SampleID, @Frame" +
"{0},@Right,@Left)", hand.skeletonData);
insertCommand = new SQLiteCommand(insertframe, connection);
insertCommand.Parameters.AddWithValue("@SampleID", CurrentSign);
insertCommand.Parameters.AddWithValue("@Frame", currentFrame);
if (hand.hogRight == null || hand.hogRight.Length == 0)
{
insertCommand.Parameters.AddWithValue("@Right", null);
}
else
{
insertCommand.Parameters.Add("@Right", DbType.Binary, hand.hogRight.Length * sizeof(float)).Value = hand.hogRight.ToByteArray();
}
if (hand.hogLeft == null || hand.hogLeft.Length == 0)
{
insertCommand.Parameters.AddWithValue("@Left", null);
}
else
{
insertCommand.Parameters.Add("@Left", DbType.Binary, hand.hogLeft.Length * sizeof(float)).Value = hand.hogLeft.ToByteArray();
}
insertCommand.ExecuteNonQuery();
string sql = @"select last_insert_rowid()";
insertCommand.CommandText = sql;
long lastId = (long)insertCommand.ExecuteScalar();
hand.frame = lastId;
// update sign type
if (hand.type == HandEnum.Intersect)
{
string update = @"UPDATE signsample
set intersected = 1
where index_id = @ID ";
SQLiteCommand updateCommand = new SQLiteCommand(update, connection);
updateCommand.Parameters.AddWithValue("@ID", CurrentSign);
//Console.WriteLine(updateCommand.CommandText);
updateCommand.ExecuteNonQuery();
}
return true;
}
catch (Exception e)
{
tran.Rollback();
return false;
}
}
private void AllFrameReady(object sender, AllFramesReadyEventArgs e)
{
//Console.Clear();
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame != null)
{
_skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
skeletonFrame.CopySkeletonDataTo(_skeletons);
Skeleton skel = _skeletons[0];
if (skel.TrackingState == SkeletonTrackingState.Tracked)
{
SkeletonPoint rightHand = _skeletons[0].Joints[JointType.HandRight].Position;
SkeletonPoint head = _skeletons[0].Joints[JointType.Head].Position;
//rightHandPosition = SkeletonPointToScreen(rightHand);
//headPosition = SkeletonPointToScreen(head);
}
}
}
using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
{
if (colorFrame != null)
{
WriteableBitmap ColorBitmap = ((SystemStatusCollection)this.DataContext).ColorBitmap;
colorFrame.CopyPixelDataTo(this._colorPixels);
((SystemStatusCollection)this.DataContext).ColorBitmap.Lock();
ColorBitmap.WritePixels(
new System.Windows.Int32Rect(0, 0, ColorBitmap.PixelWidth, ColorBitmap.PixelHeight),
_colorPixels,
ColorBitmap.PixelWidth * sizeof(int),
0);
ColorBitmap.Unlock();
}
}
using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
{
if (depthFrame != null)
{
var sw = Stopwatch.StartNew();
// Copy the pixel data from the image to a temporary array
depthFrame.CopyDepthImagePixelDataTo(_depthPixels);
// short[] depthData = new short[depthFrame.PixelDataLength];
// Get the min and max reliable depth for the current frame
int minDepth = depthFrame.MinDepth;
int maxDepth = depthFrame.MaxDepth;
int width = depthFrame.Width;
int height = depthFrame.Height;
//if (headPosition.X == 0)
//{
// headDepth = 100;
//}
//else
//{
// headDepth = depthPixels[headPosition.X + headPosition.Y * 640].Depth;
//}
//var lowDepths = depthPixels.Skip(depthPixels.Length - 640).Where(x => x.Depth > 0);
//int lowDepth = lowDepths.Count() > 0 ? lowDepths.Min(x => x.Depth) : 0;
//float a = CalTiltAngle(headPosition.Y, depthPixels[headPosition.X+headPosition.Y*640].Depth, lowDepth);
//Console.WriteLine("head depth :{0} lowDepth:{1}, TiltAngle:{2}", depthPixels[headPosition.X + headPosition.Y * 640].Depth, lowDepth);
// Console.WriteLine("get low depth:" + sw.ElapsedMilliseconds);
//sw.Restart();
//*********** Convert cull and transform*****************
//colorizer.TransformCullAndConvertDepthFrame(
// depthPixels, minDepth, maxDepth, colorPixels,
// AngleRotateTan,
// (short)(headDepth - (short)CullingThresh), headPosition);
//Image<Bgra, byte> depthImg;
////Console.WriteLine("iteration:" + sw.ElapsedMilliseconds);
//sw.Restart();
//*******find hand*****************
//Image<Gray, Byte> rightFront = null;
//Image<Gray, Byte> leftFront = null;
//depthImg = ImageConverter.Array2Image(colorPixels, width, height, width * 4);
Point rightVector = new Point();
Point leftVector = new Point();
bool isSkip = false;
bool rightHandRaise = false;
if (_skeletons != null)
{
foreach (var skel in _skeletons)
{
if (skel.TrackingState == SkeletonTrackingState.Tracked)
{
if (IsRecording)
{
capturedSkeletons.Add(skel);
}
Point hr = SkeletonPointToScreen(_skeletons[0].Joints[JointType.HandRight].Position);
Point hl = SkeletonPointToScreen(_skeletons[0].Joints[JointType.HandLeft].Position);
Point er = SkeletonPointToScreen(_skeletons[0].Joints[JointType.ElbowRight].Position);
Point el = SkeletonPointToScreen(_skeletons[0].Joints[JointType.ElbowLeft].Position);
Point hip = SkeletonPointToScreen(_skeletons[0].Joints[JointType.HipCenter].Position);
// hand is lower than hip
//Console.WriteLine(_skeletons[0].Joints[JointType.HandRight].Position.Y);
//Console.WriteLine(_skeletons[0].Joints[JointType.HipCenter].Position.Y);
//Console.WriteLine("-------------");
if (skel.Joints[JointType.HandRight].Position.Y <
skel.Joints[JointType.HipCenter].Position.Y + 0.05)
{
isSkip = true;
}
if (skel.Joints[JointType.HandRight].Position.Y >
skel.Joints[JointType.HipCenter].Position.Y)
{
rightHandRaise = true;
}
rightVector.X = (hr.X - er.X);
rightVector.Y = (hr.Y - er.Y);
leftVector.X = (hl.X - el.X);
leftVector.Y = (hl.Y - el.Y);
}
}
}
HandShapeModel handModel = new HandShapeModel(0, HandEnum.None);
}
}
}
private void AllFrameReady(object sender, AllFramesReadyEventArgs e)
{
//Console.Clear();
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame != null)
{
skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
skeletonFrame.CopySkeletonDataTo(this.skeletons);
Skeleton skel = skeletons[0];
if (skel.TrackingState == SkeletonTrackingState.Tracked)
{
SkeletonPoint rightHand = skeletons[0].Joints[JointType.HandRight].Position;
SkeletonPoint head = skeletons[0].Joints[JointType.Head].Position;
rightHandPosition = SkeletonPointToScreen(rightHand);
headPosition = SkeletonPointToScreen(head);
}
}
}
using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
{
if (colorFrame != null)
{
// Copy the pixel data from the image to a temporary array
colorFrame.CopyPixelDataTo(this.colorPixels);
// Write the pixel data into our bitmap
this.ColorWriteBitmap.WritePixels(
new System.Windows.Int32Rect(0, 0, this.ColorWriteBitmap.PixelWidth, this.ColorWriteBitmap.PixelHeight),
this.colorPixels,
this.ColorWriteBitmap.PixelWidth * sizeof(int),
0);
}
}
using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
{
if (depthFrame != null)
{
var sw = Stopwatch.StartNew();
// Copy the pixel data from the image to a temporary array
depthFrame.CopyDepthImagePixelDataTo(this.depthPixels);
// short[] depthData = new short[depthFrame.PixelDataLength];
// Get the min and max reliable depth for the current frame
int minDepth = depthFrame.MinDepth;
int maxDepth = depthFrame.MaxDepth;
int width = depthFrame.Width;
int height = depthFrame.Height;
if (headPosition.X == 0)
{
headDepth = 100;
}
else
{
headDepth = depthPixels[headPosition.X + headPosition.Y * 640].Depth;
}
sw.Restart();
//*********** Convert cull and transform*****************
colorizer.TransformCullAndConvertDepthFrame(
depthPixels, minDepth, maxDepth, colorPixels,
AngleRotateTan,
(short)(headDepth - (short)CullingThresh), headPosition);
Image<Bgra, byte> depthImg;
//Console.WriteLine("iteration:" + sw.ElapsedMilliseconds);
sw.Restart();
Image<Gray, Byte> rightFront = null;
Image<Gray, Byte> leftFront = null;
depthImg = ImageConverter.Array2Image(colorPixels, width, height, width * 4);
PointF rightVector = PointF.Empty;
PointF leftVector = PointF.Empty;
bool isSkip = false;
bool leftHandRaise = false;
if (skeletons != null && skeletons[0].TrackingState == SkeletonTrackingState.Tracked)
{
PointF hr = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandRight].Position);
PointF hl = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandLeft].Position);
PointF er = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowRight].Position);
PointF el = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowLeft].Position);
PointF hip = SkeletonPointToScreen(skeletons[0].Joints[JointType.HipCenter].Position);
// hand is lower than hip
//Console.WriteLine(skeletons[0].Joints[JointType.HandRight].Position.Y);
//Console.WriteLine(skeletons[0].Joints[JointType.HipCenter].Position.Y);
//Console.WriteLine("-------------");
if (skeletons[0].Joints[JointType.HandRight].Position.Y <
skeletons[0].Joints[JointType.HipCenter].Position.Y + 0.05)
{
isSkip = true;
}
if (skeletons[0].Joints[JointType.HandLeft].Position.Y >
skeletons[0].Joints[JointType.HipCenter].Position.Y )
{
leftHandRaise = true;
}
rightVector.X = (hr.X - er.X);
rightVector.Y = (hr.Y - er.Y);
leftVector.X = (hl.X - el.X);
leftVector.Y = (hl.Y - el.Y);
}
HandShapeModel handModel = null;
if (!isSkip)
{
//handModel = m_OpenCVController.FindHandPart(ref depthImg, out rightFront, out leftFront, headDepth - (int)CullingThresh, rightVector, leftVector,leftHandRaise);
}
// no hands detected
if (handModel == null)
{
handModel = new HandShapeModel(0, HandEnum.None);
}
//sw.Restart();
// database processing
DBManager db = DBManager.GetSingleton();
if (db != null)
{
if (skeletons != null)
{
handModel.SetSkeletonData(skeletons[0]);
}
db.AddFrameData(handModel);
}
// not recording show prob
else
{
//Image<Bgr, byte>[] result = HandShapeClassifier.GetSingleton()
//.RecognizeGesture(handModel.hogRight, 3);
////Console.WriteLine(sw.ElapsedMilliseconds);
//if (result != null)
//{
// ImageConverter.UpdateWriteBMP(WrtBMP_Candidate1, result[0].Convert<Gray, byte>().ToBitmap());
// ImageConverter.UpdateWriteBMP(WrtBMP_Candidate2, result[1].Convert<Gray, byte>().ToBitmap());
// ImageConverter.UpdateWriteBMP(WrtBMP_Candidate3, result[2].Convert<Gray, byte>().ToBitmap());
//}
}
string currentSign = db == null ? "0" : db.CurrentSign.ToString();
string path = @"J:\Kinect data\Aaron 141-180\hands\" + currentSign + " " + handModel.frame.ToString();
// UI update
if (rightFront != null)
{
Bitmap right = rightFront.ToBitmap();
//right.Save(path + " r.jpg");
ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, right);
}
if (leftFront != null)
{
Bitmap left = leftFront.ToBitmap();
//left.Save(path + " l.jpg");
ImageConverter.UpdateWriteBMP(WrtBMP_LeftHandFront, left);
}
if (sw.ElapsedMilliseconds>15)
{
Console.WriteLine("Find hand:" + sw.ElapsedMilliseconds);
}
sw.Restart();
//**************************draw gray histogram
//Bitmap histo = m_OpenCVController.Histogram(depthImg);
//ImageConverter.UpdateWriteBMP(GrayWriteBitmap, histo);
// draw hand position from kinect
// DrawHandPosition(depthBMP, rightHandPosition, System.Drawing.Brushes.Purple);
//*******************upadte UI
ImageConverter.UpdateWriteBMP(DepthWriteBitmap, depthImg.ToBitmap());
// Console.WriteLine("Update UI:" + sw.ElapsedMilliseconds);
}
}
}
public void WriteFrameData(HandShapeModel handModel)
{
if (connection == null || command == null)
{
return;
}
// cmd = String.Format("INSERT INTO FrameData " +
// "(FrameDataId, SignInfoId, FrameCount, OffsetedFrameCount, " + //0, 1, 2
//"SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " +
//"SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " +
//"SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " +
//"SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " +
//"SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " +
//"SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " +
//"SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " +
//"SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " +
//"SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " +
//"SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " +
//"SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " +
//"SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " +
//"SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " +
//"SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " +
// "HandCount, " +
// "Hand0FingertipCount, " +
// "Hand0Fingertip0X, Hand0Fingertip0Y, Hand0Fingertip0Z, " +
// "Hand0Fingertip1X, Hand0Fingertip1Y, Hand0Fingertip1Z, " +
// "Hand0Fingertip2X, Hand0Fingertip2Y, Hand0Fingertip2Z, " +
// "Hand0Fingertip3X, Hand0Fingertip3Y, Hand0Fingertip3Z, " +
// "Hand0Fingertip4X, Hand0Fingertip4Y, Hand0Fingertip4Z, " +
// "Hand0EllipseCenterX, Hand0EllipseCenterY, " +
// "Hand0EllipseMajorAxis, Hand0EllipseMinorAxis, " +
// "Hand0EllipseAspectRatio, Hand0AxisTheta, " +
// "Hand1FingertipCount, " +
// "Hand1Fingertip0X, Hand1Fingertip0Y, Hand1Fingertip0Z, " +
// "Hand1Fingertip1X, Hand1Fingertip1Y, Hand1Fingertip1Z, " +
// "Hand1Fingertip2X, Hand1Fingertip2Y, Hand1Fingertip2Z, " +
// "Hand1Fingertip3X, Hand1Fingertip3Y, Hand1Fingertip3Z, " +
// "Hand1Fingertip4X, Hand1Fingertip4Y, Hand1Fingertip4Z, " +
// "Hand1EllipseCenterX, Hand1EllipseCenterY, " +
// "Hand1EllipseMajorAxis, Hand1EllipseMinorAxis, " +
// "Hand1EllipseAspectRatio, Hand1AxisTheta) " +
// "VALUES(NULL, {0}, {1}, {2}{3})",
// SignInfoId, frameData.frameCount, frameData.OffsetedFrameCount, frameData.GetFrameDataArgString()
// );
// insertions.Add(cmd);
//} else {
// if (insertions.Count > 0) { //Batch insertion
// var sw = Stopwatch.StartNew();
// reader.Close();
// SQLiteTransaction trans = connection.BeginTransaction(); // <-------------------
// try
// {
// foreach (string insertion in insertions)
// {
// command.CommandText = insertion;
// command.ExecuteNonQuery();
// }
// insertions.Clear();
// trans.Commit(); // <-------------------
// }
// catch
// {
// trans.Rollback(); // <-------------------
// throw; // <-------------------
// }
// Console.WriteLine("save data:" + sw.ElapsedMilliseconds);
// }
//}
//if (!reader.IsClosed) {
// reader.Close();
//}
////if (cmd != String.Empty) {
//// command.CommandText = cmd;
//// command.ExecuteNonQuery();
////}
}
public bool AddFrameData(HandShapeModel hand)
{
currentFrame++;
if (!Begin || hand == null)
{
return(false);
}
try
{
SQLiteCommand insertCommand = new SQLiteCommand(connection);
string insertframe =
String.Format("INSERT OR REPLACE INTO FrameData (" +
"SampleIndex,FrameNumber," +
"SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " +
"SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " +
"SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " +
"SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " +
"SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " +
"SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " +
"SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " +
"SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " +
"SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " +
"SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " +
"SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " +
"SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " +
"SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " +
"SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " +
"RightHandHOG, LeftHandHOG) VALUES" +
"(@SampleID, @Frame" +
"{0},@Right,@Left)", hand.skeletonData);
insertCommand = new SQLiteCommand(insertframe, connection);
insertCommand.Parameters.AddWithValue("@SampleID", CurrentSign);
insertCommand.Parameters.AddWithValue("@Frame", currentFrame);
if (hand.hogRight == null || hand.hogRight.Length == 0)
{
insertCommand.Parameters.AddWithValue("@Right", null);
}
else
{
insertCommand.Parameters.Add("@Right", DbType.Binary, hand.hogRight.Length * sizeof(float)).Value = hand.hogRight.ToByteArray();
}
if (hand.hogLeft == null || hand.hogLeft.Length == 0)
{
insertCommand.Parameters.AddWithValue("@Left", null);
}
else
{
insertCommand.Parameters.Add("@Left", DbType.Binary, hand.hogLeft.Length * sizeof(float)).Value = hand.hogLeft.ToByteArray();
}
insertCommand.ExecuteNonQuery();
string sql = @"select last_insert_rowid()";
insertCommand.CommandText = sql;
long lastId = (long)insertCommand.ExecuteScalar();
hand.frame = lastId;
// update sign type
if (hand.type == HandEnum.Intersect)
{
string update = @"UPDATE signsample
set intersected = 1
where index_id = @ID ";
SQLiteCommand updateCommand = new SQLiteCommand(update, connection);
updateCommand.Parameters.AddWithValue("@ID", CurrentSign);
//Console.WriteLine(updateCommand.CommandText);
updateCommand.ExecuteNonQuery();
}
return(true);
}
catch (Exception e)
{
tran.Rollback();
return(false);
}
}
