public bool ProcessFrame() { if (CurrentFrame >= TotalFrames - 1) { return(false); } if (_CCapture != null) { var r = _CCapture.Grab(); var imgc = _CCapture.RetrieveBgrFrame(); ImageConverter.UpdateWriteBMP(colorWriteBitmap, imgc.ToBitmap()); } if (_DCapture != null) { var r = _DCapture.Grab(); var img = _DCapture.RetrieveBgrFrame(); int framenumber = (int)_DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_FRAMES); //Console.WriteLine("frame:"+framenumber.ToString()); CurrentFrame = framenumber - 1; if (img == null) { return(false); } //Show time stamp double time_index = _DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_MSEC); PointF rightVector = new PointF(-10, -10); PointF leftVector = new PointF(10, -10); bool isSkip = false; bool leftHandRaise = false; if (CurrentFrame >= sktList.Count || !sktList[CurrentFrame].Tracked) { // no skeleton detected headDepth = 0; } else { headPosition = sktList[CurrentFrame][MyJointType.Head].PosDepth; headDepth = Math.Min((int)img[headPosition.Y, headPosition.X].Green, (int)img[headPosition.Y, headPosition.X].Red); headDepth = Math.Min(headDepth, (int)img[headPosition.Y, headPosition.X].Blue); PointF hr = sktList[CurrentFrame][MyJointType.HandR].PosDepth; PointF hl = sktList[CurrentFrame][MyJointType.HandL].PosDepth; PointF er = sktList[CurrentFrame][MyJointType.ElbowR].PosDepth; PointF el = sktList[CurrentFrame][MyJointType.ElbowL].PosDepth; PointF hip = sktList[CurrentFrame][MyJointType.HipCenter].PosDepth; // hand is lower than hip //Console.WriteLine(sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y); //Console.WriteLine(sktList[CurrentFrame][MyJointType.ElbowR].Pos3D.Y); //Console.WriteLine(sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y); //Console.WriteLine("-------------"); if (sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y < sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y + 0.05) { isSkip = true; } if (sktList[CurrentFrame][MyJointType.HandL].Pos3D.Y > -0.02f) { leftHandRaise = true; } rightVector.X = (hr.X - er.X); rightVector.Y = (hr.Y - er.Y); leftVector.X = (hl.X - el.X); leftVector.Y = (hl.Y - el.Y); } #region temp //isSkip = false; //leftHandRaise = false; //rightVector = new PointF(-10, -10); //leftVector = new PointF(10, -10); //headDepth = (int)img[85, 315].Blue; #endregion // Console.WriteLine("headdepth:"+headDepth.ToString()); //***********cull image***************** double cull = headDepth - CullingThresh; var depthImg = img.ThresholdToZeroInv(new Bgr(cull, cull, cull)); //Image<Gray, Byte> depthImg = img.Convert<Gray, byte>().ThresholdBinary(new Gray(160), new Gray(255)); var sw = Stopwatch.StartNew(); int handDepth = (int)(2600.0 / 255 * cull + 400); HandShapeModel handModel = null; Image <Gray, Byte> rightFront = null; Image <Gray, Byte> leftFront = null; // isskip is invalid coz no hip data isSkip = false; if (cull > 0) { handModel = m_opencv.FindHandPart(ref depthImg, out rightFront, out leftFront, handDepth, rightVector, leftVector, leftHandRaise); } viewer.Image = depthImg; // no hands detected if (handModel == null) { handModel = new HandShapeModel(0, HandEnum.None); } var row = new CsvRow(); row.Add(CurrentFrame.ToString()); row.Add(handModel.type.ToString()); switch (handModel.type) { case HandEnum.Intersect: case HandEnum.Right: row.Add(handModel.handPosRight.center.X.ToString()); row.Add(handModel.handPosRight.center.Y.ToString()); row.Add(handModel.handPosRight.size.Width.ToString()); row.Add(handModel.handPosRight.size.Height.ToString()); row.Add(handModel.handPosRight.angle.ToString()); break; case HandEnum.Both: row.Add(handModel.handPosRight.center.X.ToString()); row.Add(handModel.handPosRight.center.Y.ToString()); row.Add(handModel.handPosRight.size.Width.ToString()); row.Add(handModel.handPosRight.size.Height.ToString()); row.Add(handModel.handPosRight.angle.ToString()); row.Add(handModel.handPosLeft.center.X.ToString()); row.Add(handModel.handPosLeft.center.Y.ToString()); row.Add(handModel.handPosLeft.size.Width.ToString()); row.Add(handModel.handPosLeft.size.Height.ToString()); row.Add(handModel.handPosLeft.angle.ToString()); break; } row.Add(cull.ToString()); labelWriter.WriteRow(row); if (rightFront != null) { ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, rightFront.ToBitmap()); } ImageConverter.UpdateWriteBMP(depthWriteBitmap, img.ToBitmap()); if (CurrentFrame < TotalFrames - 1) { return(true); } } return(false); }
public bool ProcessFrame() { if (CurrentFrame >= TotalFrames -1) { return false; } if (_CCapture != null) { var r = _CCapture.Grab(); var imgc = _CCapture.RetrieveBgrFrame(); ImageConverter.UpdateWriteBMP(colorWriteBitmap, imgc.ToBitmap()); } if (_DCapture != null) { var r = _DCapture.Grab(); var img = _DCapture.RetrieveBgrFrame(); int framenumber = (int)_DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_FRAMES); //Console.WriteLine("frame:"+framenumber.ToString()); CurrentFrame = framenumber - 1; if (img == null) { return false; } //Show time stamp double time_index = _DCapture.GetCaptureProperty(Emgu.CV.CvEnum.CAP_PROP.CV_CAP_PROP_POS_MSEC); PointF rightVector = new PointF(-10, -10); PointF leftVector = new PointF(10, -10); bool isSkip = false; bool leftHandRaise = false; if (CurrentFrame >= sktList.Count || !sktList[CurrentFrame].Tracked) { // no skeleton detected headDepth = 0; } else { headPosition = sktList[CurrentFrame][MyJointType.Head].PosDepth; headDepth = Math.Min((int)img[headPosition.Y, headPosition.X].Green,(int)img[headPosition.Y, headPosition.X].Red); headDepth = Math.Min(headDepth, (int) img[headPosition.Y, headPosition.X].Blue); PointF hr = sktList[CurrentFrame][MyJointType.HandR].PosDepth; PointF hl = sktList[CurrentFrame][MyJointType.HandL].PosDepth; PointF er = sktList[CurrentFrame][MyJointType.ElbowR].PosDepth; PointF el = sktList[CurrentFrame][MyJointType.ElbowL].PosDepth; PointF hip = sktList[CurrentFrame][MyJointType.HipCenter].PosDepth; // hand is lower than hip //Console.WriteLine(sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y); //Console.WriteLine(sktList[CurrentFrame][MyJointType.ElbowR].Pos3D.Y); //Console.WriteLine(sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y); //Console.WriteLine("-------------"); if (sktList[CurrentFrame][MyJointType.HandR].Pos3D.Y < sktList[CurrentFrame][MyJointType.HipCenter].Pos3D.Y + 0.05) { isSkip = true; } if (sktList[CurrentFrame][MyJointType.HandL].Pos3D.Y > -0.02f) { leftHandRaise = true; } rightVector.X = (hr.X - er.X); rightVector.Y = (hr.Y - er.Y); leftVector.X = (hl.X - el.X); leftVector.Y = (hl.Y - el.Y); } #region temp //isSkip = false; //leftHandRaise = false; //rightVector = new PointF(-10, -10); //leftVector = new PointF(10, -10); //headDepth = (int)img[85, 315].Blue; #endregion // Console.WriteLine("headdepth:"+headDepth.ToString()); //***********cull image***************** double cull = headDepth - CullingThresh; var depthImg = img.ThresholdToZeroInv(new Bgr(cull, cull, cull)); //Image<Gray, Byte> depthImg = img.Convert<Gray, byte>().ThresholdBinary(new Gray(160), new Gray(255)); var sw = Stopwatch.StartNew(); int handDepth = (int)(2600.0 / 255 * cull + 400); HandShapeModel handModel = null; Image<Gray, Byte> rightFront = null; Image<Gray, Byte> leftFront = null; // isskip is invalid coz no hip data isSkip = false; if (cull> 0) { handModel = m_opencv.FindHandPart(ref depthImg, out rightFront, out leftFront, handDepth, rightVector, leftVector, leftHandRaise); } viewer.Image = depthImg; // no hands detected if (handModel == null) { handModel = new HandShapeModel(0, HandEnum.None); } var row = new CsvRow(); row.Add(CurrentFrame.ToString()); row.Add(handModel.type.ToString()); switch (handModel.type) { case HandEnum.Intersect: case HandEnum.Right: row.Add(handModel.handPosRight.center.X.ToString()); row.Add(handModel.handPosRight.center.Y.ToString()); row.Add(handModel.handPosRight.size.Width.ToString()); row.Add(handModel.handPosRight.size.Height.ToString()); row.Add(handModel.handPosRight.angle.ToString()); break; case HandEnum.Both: row.Add(handModel.handPosRight.center.X.ToString()); row.Add(handModel.handPosRight.center.Y.ToString()); row.Add(handModel.handPosRight.size.Width.ToString()); row.Add(handModel.handPosRight.size.Height.ToString()); row.Add(handModel.handPosRight.angle.ToString()); row.Add(handModel.handPosLeft.center.X.ToString()); row.Add(handModel.handPosLeft.center.Y.ToString()); row.Add(handModel.handPosLeft.size.Width.ToString()); row.Add(handModel.handPosLeft.size.Height.ToString()); row.Add(handModel.handPosLeft.angle.ToString()); break; } row.Add(cull.ToString()); labelWriter.WriteRow(row); if (rightFront != null) { ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, rightFront.ToBitmap()); } ImageConverter.UpdateWriteBMP(depthWriteBitmap, img.ToBitmap()); if (CurrentFrame < TotalFrames -1) { return true; } } return false; }
public unsafe HandShapeModel FindHandPart( ref Image<Bgr, byte> image, out Image<Gray, Byte> rightFront, out Image<Gray, Byte> leftFront, int handDepth, PointF rightVector, PointF leftVector, bool leftHandRaise ) { //process hand size if (handDepth > 0) { begin = (int)(100.0 * 240.0 / handDepth / 0.39); end = (int)(220 * 240.0 / handDepth / 0.39); minLength = (int)(150 * 240.0 / handDepth / 0.39); } else { rightFront = null; leftFront = null; return null; } //Console.WriteLine(begin); //Console.WriteLine(end); //Console.WriteLine(minLength); rightFront = null; leftFront = null; HandShapeModel model = null; //find contour grayImg = image.Convert<Gray, byte>(); Image<Gray, Byte> binaryImg = GetBinaryImg(image); image = binaryImg.Convert<Bgr, byte>(); List<MCvBox2D> rectList = FindContourMBox(binaryImg); //draw contour foreach (var rect in rectList) { DrawPoly(rect.GetVertices().ToPoints(), image, new MCvScalar(255, 0, 0)); } // find hand rectList = rectList.OrderByDescending(x => x.GetTrueArea()).ToList(); MCvBox2D rightHand; MCvBox2D leftHand; Font textFont = new Font(FontFamily.Families[0], 20); // count hands number using (Graphics g = Graphics.FromImage(image.Bitmap)) { // 3 conditions in total if (rectList.Count() >= 2)//two hands { if (rectList[0].center.X > rectList[1].center.X) { rightHand = rectList[0]; leftHand = rectList[1]; } else { rightHand = rectList[1]; leftHand = rectList[0]; } // mark intersect state Intersect = rightHand.MinAreaRect().IsCloseTo(leftHand.MinAreaRect(), 5); //right hand MCvBox2D SplittedRightHand = SplitHand(rightHand, HandEnum.Right, rightVector); rightFront = GetSubImage<Gray>(binaryImg, SplittedRightHand, rightHand.angle); float[] rightHog = CalHog(rightFront); DrawHand(SplittedRightHand, image, HandEnum.Right); //left hand MCvBox2D SplittedLeftHand = SplitHand(leftHand, HandEnum.Left, leftVector); leftFront = GetSubImage<Gray>(binaryImg, SplittedLeftHand, leftHand.angle); float[] leftHog = CalHog(leftFront); DrawHand(SplittedLeftHand, image, HandEnum.Left); g.DrawString("left and right", textFont, Brushes.Red, 0, 20); model = new HandShapeModel(hogSize, HandEnum.Both); model.hogLeft = leftHog; model.hogRight = rightHog; model.handPosRight = SplittedRightHand; model.handPosLeft = SplittedLeftHand; } else if (rectList.Count() == 1) // one rectangle { string text = ""; leftFront = null; if (leftHandRaise || Intersect) { text = "Two hands"; var recs = FindContourRect(binaryImg); if (recs.Count > 0) { var rec = recs[0]; rightFront = GetSubImageByRect<Gray>(binaryImg, rec); MCvBox2D box = rec.ToCvBox2D(); DrawHand(box, image, HandEnum.Intersect); float[] TwoHandHOG = CalHog(rightFront); model = new HandShapeModel(hogSize, HandEnum.Intersect); model.hogRight = TwoHandHOG; model.handPosRight = box; } } else { text = "right"; MCvBox2D SplittedRightHand = SplitHand(rectList[0], HandEnum.Right, rightVector); //Console.WriteLine(SplittedRightHand.size.Height); rightFront = GetSubImage<Gray>(binaryImg, SplittedRightHand, rectList[0].angle); DrawHand(SplittedRightHand, image, HandEnum.Right); float[] TwoHandHOG = CalHog(rightFront); model = new HandShapeModel(hogSize, HandEnum.Right); model.hogRight = TwoHandHOG; model.handPosRight = SplittedRightHand; } g.DrawString(text, textFont, Brushes.Red, 0, 20); } else // no hand detected { Intersect = false; } } return model; }
public void WriteFrameData(HandShapeModel handModel) { if (connection == null || command == null) return; // cmd = String.Format("INSERT INTO FrameData " + // "(FrameDataId, SignInfoId, FrameCount, OffsetedFrameCount, " + //0, 1, 2 //"SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " + //"SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " + //"SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " + //"SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " + //"SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " + //"SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " + //"SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " + //"SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " + //"SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " + //"SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " + //"SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " + //"SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " + //"SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " + //"SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " + // "HandCount, " + // "Hand0FingertipCount, " + // "Hand0Fingertip0X, Hand0Fingertip0Y, Hand0Fingertip0Z, " + // "Hand0Fingertip1X, Hand0Fingertip1Y, Hand0Fingertip1Z, " + // "Hand0Fingertip2X, Hand0Fingertip2Y, Hand0Fingertip2Z, " + // "Hand0Fingertip3X, Hand0Fingertip3Y, Hand0Fingertip3Z, " + // "Hand0Fingertip4X, Hand0Fingertip4Y, Hand0Fingertip4Z, " + // "Hand0EllipseCenterX, Hand0EllipseCenterY, " + // "Hand0EllipseMajorAxis, Hand0EllipseMinorAxis, " + // "Hand0EllipseAspectRatio, Hand0AxisTheta, " + // "Hand1FingertipCount, " + // "Hand1Fingertip0X, Hand1Fingertip0Y, Hand1Fingertip0Z, " + // "Hand1Fingertip1X, Hand1Fingertip1Y, Hand1Fingertip1Z, " + // "Hand1Fingertip2X, Hand1Fingertip2Y, Hand1Fingertip2Z, " + // "Hand1Fingertip3X, Hand1Fingertip3Y, Hand1Fingertip3Z, " + // "Hand1Fingertip4X, Hand1Fingertip4Y, Hand1Fingertip4Z, " + // "Hand1EllipseCenterX, Hand1EllipseCenterY, " + // "Hand1EllipseMajorAxis, Hand1EllipseMinorAxis, " + // "Hand1EllipseAspectRatio, Hand1AxisTheta) " + // "VALUES(NULL, {0}, {1}, {2}{3})", // SignInfoId, frameData.frameCount, frameData.OffsetedFrameCount, frameData.GetFrameDataArgString() // ); // insertions.Add(cmd); //} else { // if (insertions.Count > 0) { //Batch insertion // var sw = Stopwatch.StartNew(); // reader.Close(); // SQLiteTransaction trans = connection.BeginTransaction(); // <------------------- // try // { // foreach (string insertion in insertions) // { // command.CommandText = insertion; // command.ExecuteNonQuery(); // } // insertions.Clear(); // trans.Commit(); // <------------------- // } // catch // { // trans.Rollback(); // <------------------- // throw; // <------------------- // } // Console.WriteLine("save data:" + sw.ElapsedMilliseconds); // } //} //if (!reader.IsClosed) { // reader.Close(); //} ////if (cmd != String.Empty) { //// command.CommandText = cmd; //// command.ExecuteNonQuery(); ////} }
private void AllFrameReady(object sender, AllFramesReadyEventArgs e) { //Console.Clear(); using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { if (skeletonFrame != null) { skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength]; skeletonFrame.CopySkeletonDataTo(this.skeletons); Skeleton skel = skeletons[0]; if (skel.TrackingState == SkeletonTrackingState.Tracked) { SkeletonPoint rightHand = skeletons[0].Joints[JointType.HandRight].Position; SkeletonPoint head = skeletons[0].Joints[JointType.Head].Position; rightHandPosition = SkeletonPointToScreen(rightHand); headPosition = SkeletonPointToScreen(head); } } } using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (colorFrame != null) { // Copy the pixel data from the image to a temporary array colorFrame.CopyPixelDataTo(this.colorPixels); // Write the pixel data into our bitmap this.ColorWriteBitmap.WritePixels( new System.Windows.Int32Rect(0, 0, this.ColorWriteBitmap.PixelWidth, this.ColorWriteBitmap.PixelHeight), this.colorPixels, this.ColorWriteBitmap.PixelWidth * sizeof(int), 0); } } using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame != null) { var sw = Stopwatch.StartNew(); // Copy the pixel data from the image to a temporary array depthFrame.CopyDepthImagePixelDataTo(this.depthPixels); // short[] depthData = new short[depthFrame.PixelDataLength]; // Get the min and max reliable depth for the current frame int minDepth = depthFrame.MinDepth; int maxDepth = depthFrame.MaxDepth; int width = depthFrame.Width; int height = depthFrame.Height; if (headPosition.X == 0) { headDepth = 100; } else { headDepth = depthPixels[headPosition.X + headPosition.Y * 640].Depth; } sw.Restart(); //*********** Convert cull and transform***************** colorizer.TransformCullAndConvertDepthFrame( depthPixels, minDepth, maxDepth, colorPixels, AngleRotateTan, (short)(headDepth - (short)CullingThresh), headPosition); Image <Bgra, byte> depthImg; //Console.WriteLine("iteration:" + sw.ElapsedMilliseconds); sw.Restart(); Image <Gray, Byte> rightFront = null; Image <Gray, Byte> leftFront = null; depthImg = ImageConverter.Array2Image(colorPixels, width, height, width * 4); PointF rightVector = PointF.Empty; PointF leftVector = PointF.Empty; bool isSkip = false; bool leftHandRaise = false; if (skeletons != null && skeletons[0].TrackingState == SkeletonTrackingState.Tracked) { PointF hr = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandRight].Position); PointF hl = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandLeft].Position); PointF er = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowRight].Position); PointF el = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowLeft].Position); PointF hip = SkeletonPointToScreen(skeletons[0].Joints[JointType.HipCenter].Position); // hand is lower than hip //Console.WriteLine(skeletons[0].Joints[JointType.HandRight].Position.Y); //Console.WriteLine(skeletons[0].Joints[JointType.HipCenter].Position.Y); //Console.WriteLine("-------------"); if (skeletons[0].Joints[JointType.HandRight].Position.Y < skeletons[0].Joints[JointType.HipCenter].Position.Y + 0.05) { isSkip = true; } if (skeletons[0].Joints[JointType.HandLeft].Position.Y > skeletons[0].Joints[JointType.HipCenter].Position.Y) { leftHandRaise = true; } rightVector.X = (hr.X - er.X); rightVector.Y = (hr.Y - er.Y); leftVector.X = (hl.X - el.X); leftVector.Y = (hl.Y - el.Y); } HandShapeModel handModel = null; if (!isSkip) { //handModel = m_OpenCVController.FindHandPart(ref depthImg, out rightFront, out leftFront, headDepth - (int)CullingThresh, rightVector, leftVector,leftHandRaise); } // no hands detected if (handModel == null) { handModel = new HandShapeModel(0, HandEnum.None); } //sw.Restart(); // database processing DBManager db = DBManager.GetSingleton(); if (db != null) { if (skeletons != null) { handModel.SetSkeletonData(skeletons[0]); } db.AddFrameData(handModel); } // not recording show prob else { //Image<Bgr, byte>[] result = HandShapeClassifier.GetSingleton() //.RecognizeGesture(handModel.hogRight, 3); ////Console.WriteLine(sw.ElapsedMilliseconds); //if (result != null) //{ // ImageConverter.UpdateWriteBMP(WrtBMP_Candidate1, result[0].Convert<Gray, byte>().ToBitmap()); // ImageConverter.UpdateWriteBMP(WrtBMP_Candidate2, result[1].Convert<Gray, byte>().ToBitmap()); // ImageConverter.UpdateWriteBMP(WrtBMP_Candidate3, result[2].Convert<Gray, byte>().ToBitmap()); //} } string currentSign = db == null ? "0" : db.CurrentSign.ToString(); string path = @"J:\Kinect data\Aaron 141-180\hands\" + currentSign + " " + handModel.frame.ToString(); // UI update if (rightFront != null) { Bitmap right = rightFront.ToBitmap(); //right.Save(path + " r.jpg"); ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, right); } if (leftFront != null) { Bitmap left = leftFront.ToBitmap(); //left.Save(path + " l.jpg"); ImageConverter.UpdateWriteBMP(WrtBMP_LeftHandFront, left); } if (sw.ElapsedMilliseconds > 15) { Console.WriteLine("Find hand:" + sw.ElapsedMilliseconds); } sw.Restart(); //**************************draw gray histogram //Bitmap histo = m_OpenCVController.Histogram(depthImg); //ImageConverter.UpdateWriteBMP(GrayWriteBitmap, histo); // draw hand position from kinect // DrawHandPosition(depthBMP, rightHandPosition, System.Drawing.Brushes.Purple); //*******************upadte UI ImageConverter.UpdateWriteBMP(DepthWriteBitmap, depthImg.ToBitmap()); // Console.WriteLine("Update UI:" + sw.ElapsedMilliseconds); } } }
public bool AddFrameData(HandShapeModel hand) { currentFrame++; if (!Begin || hand == null) { return false; } try { SQLiteCommand insertCommand = new SQLiteCommand(connection); string insertframe = String.Format("INSERT OR REPLACE INTO FrameData (" + "SampleIndex,FrameNumber," + "SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " + "SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " + "SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " + "SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " + "SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " + "SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " + "SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " + "SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " + "SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " + "SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " + "SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " + "SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " + "SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " + "SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " + "RightHandHOG, LeftHandHOG) VALUES" + "(@SampleID, @Frame" + "{0},@Right,@Left)", hand.skeletonData); insertCommand = new SQLiteCommand(insertframe, connection); insertCommand.Parameters.AddWithValue("@SampleID", CurrentSign); insertCommand.Parameters.AddWithValue("@Frame", currentFrame); if (hand.hogRight == null || hand.hogRight.Length == 0) { insertCommand.Parameters.AddWithValue("@Right", null); } else { insertCommand.Parameters.Add("@Right", DbType.Binary, hand.hogRight.Length * sizeof(float)).Value = hand.hogRight.ToByteArray(); } if (hand.hogLeft == null || hand.hogLeft.Length == 0) { insertCommand.Parameters.AddWithValue("@Left", null); } else { insertCommand.Parameters.Add("@Left", DbType.Binary, hand.hogLeft.Length * sizeof(float)).Value = hand.hogLeft.ToByteArray(); } insertCommand.ExecuteNonQuery(); string sql = @"select last_insert_rowid()"; insertCommand.CommandText = sql; long lastId = (long)insertCommand.ExecuteScalar(); hand.frame = lastId; // update sign type if (hand.type == HandEnum.Intersect) { string update = @"UPDATE signsample set intersected = 1 where index_id = @ID "; SQLiteCommand updateCommand = new SQLiteCommand(update, connection); updateCommand.Parameters.AddWithValue("@ID", CurrentSign); //Console.WriteLine(updateCommand.CommandText); updateCommand.ExecuteNonQuery(); } return true; } catch (Exception e) { tran.Rollback(); return false; } }
private void AllFrameReady(object sender, AllFramesReadyEventArgs e) { //Console.Clear(); using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { if (skeletonFrame != null) { _skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength]; skeletonFrame.CopySkeletonDataTo(_skeletons); Skeleton skel = _skeletons[0]; if (skel.TrackingState == SkeletonTrackingState.Tracked) { SkeletonPoint rightHand = _skeletons[0].Joints[JointType.HandRight].Position; SkeletonPoint head = _skeletons[0].Joints[JointType.Head].Position; //rightHandPosition = SkeletonPointToScreen(rightHand); //headPosition = SkeletonPointToScreen(head); } } } using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (colorFrame != null) { WriteableBitmap ColorBitmap = ((SystemStatusCollection)this.DataContext).ColorBitmap; colorFrame.CopyPixelDataTo(this._colorPixels); ((SystemStatusCollection)this.DataContext).ColorBitmap.Lock(); ColorBitmap.WritePixels( new System.Windows.Int32Rect(0, 0, ColorBitmap.PixelWidth, ColorBitmap.PixelHeight), _colorPixels, ColorBitmap.PixelWidth * sizeof(int), 0); ColorBitmap.Unlock(); } } using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame != null) { var sw = Stopwatch.StartNew(); // Copy the pixel data from the image to a temporary array depthFrame.CopyDepthImagePixelDataTo(_depthPixels); // short[] depthData = new short[depthFrame.PixelDataLength]; // Get the min and max reliable depth for the current frame int minDepth = depthFrame.MinDepth; int maxDepth = depthFrame.MaxDepth; int width = depthFrame.Width; int height = depthFrame.Height; //if (headPosition.X == 0) //{ // headDepth = 100; //} //else //{ // headDepth = depthPixels[headPosition.X + headPosition.Y * 640].Depth; //} //var lowDepths = depthPixels.Skip(depthPixels.Length - 640).Where(x => x.Depth > 0); //int lowDepth = lowDepths.Count() > 0 ? lowDepths.Min(x => x.Depth) : 0; //float a = CalTiltAngle(headPosition.Y, depthPixels[headPosition.X+headPosition.Y*640].Depth, lowDepth); //Console.WriteLine("head depth :{0} lowDepth:{1}, TiltAngle:{2}", depthPixels[headPosition.X + headPosition.Y * 640].Depth, lowDepth); // Console.WriteLine("get low depth:" + sw.ElapsedMilliseconds); //sw.Restart(); //*********** Convert cull and transform***************** //colorizer.TransformCullAndConvertDepthFrame( // depthPixels, minDepth, maxDepth, colorPixels, // AngleRotateTan, // (short)(headDepth - (short)CullingThresh), headPosition); //Image<Bgra, byte> depthImg; ////Console.WriteLine("iteration:" + sw.ElapsedMilliseconds); //sw.Restart(); //*******find hand***************** //Image<Gray, Byte> rightFront = null; //Image<Gray, Byte> leftFront = null; //depthImg = ImageConverter.Array2Image(colorPixels, width, height, width * 4); Point rightVector = new Point(); Point leftVector = new Point(); bool isSkip = false; bool rightHandRaise = false; if (_skeletons != null) { foreach (var skel in _skeletons) { if (skel.TrackingState == SkeletonTrackingState.Tracked) { if (IsRecording) { capturedSkeletons.Add(skel); } Point hr = SkeletonPointToScreen(_skeletons[0].Joints[JointType.HandRight].Position); Point hl = SkeletonPointToScreen(_skeletons[0].Joints[JointType.HandLeft].Position); Point er = SkeletonPointToScreen(_skeletons[0].Joints[JointType.ElbowRight].Position); Point el = SkeletonPointToScreen(_skeletons[0].Joints[JointType.ElbowLeft].Position); Point hip = SkeletonPointToScreen(_skeletons[0].Joints[JointType.HipCenter].Position); // hand is lower than hip //Console.WriteLine(_skeletons[0].Joints[JointType.HandRight].Position.Y); //Console.WriteLine(_skeletons[0].Joints[JointType.HipCenter].Position.Y); //Console.WriteLine("-------------"); if (skel.Joints[JointType.HandRight].Position.Y < skel.Joints[JointType.HipCenter].Position.Y + 0.05) { isSkip = true; } if (skel.Joints[JointType.HandRight].Position.Y > skel.Joints[JointType.HipCenter].Position.Y) { rightHandRaise = true; } rightVector.X = (hr.X - er.X); rightVector.Y = (hr.Y - er.Y); leftVector.X = (hl.X - el.X); leftVector.Y = (hl.Y - el.Y); } } } HandShapeModel handModel = new HandShapeModel(0, HandEnum.None); } } }
private void AllFrameReady(object sender, AllFramesReadyEventArgs e) { //Console.Clear(); using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { if (skeletonFrame != null) { skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength]; skeletonFrame.CopySkeletonDataTo(this.skeletons); Skeleton skel = skeletons[0]; if (skel.TrackingState == SkeletonTrackingState.Tracked) { SkeletonPoint rightHand = skeletons[0].Joints[JointType.HandRight].Position; SkeletonPoint head = skeletons[0].Joints[JointType.Head].Position; rightHandPosition = SkeletonPointToScreen(rightHand); headPosition = SkeletonPointToScreen(head); } } } using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (colorFrame != null) { // Copy the pixel data from the image to a temporary array colorFrame.CopyPixelDataTo(this.colorPixels); // Write the pixel data into our bitmap this.ColorWriteBitmap.WritePixels( new System.Windows.Int32Rect(0, 0, this.ColorWriteBitmap.PixelWidth, this.ColorWriteBitmap.PixelHeight), this.colorPixels, this.ColorWriteBitmap.PixelWidth * sizeof(int), 0); } } using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame != null) { var sw = Stopwatch.StartNew(); // Copy the pixel data from the image to a temporary array depthFrame.CopyDepthImagePixelDataTo(this.depthPixels); // short[] depthData = new short[depthFrame.PixelDataLength]; // Get the min and max reliable depth for the current frame int minDepth = depthFrame.MinDepth; int maxDepth = depthFrame.MaxDepth; int width = depthFrame.Width; int height = depthFrame.Height; if (headPosition.X == 0) { headDepth = 100; } else { headDepth = depthPixels[headPosition.X + headPosition.Y * 640].Depth; } sw.Restart(); //*********** Convert cull and transform***************** colorizer.TransformCullAndConvertDepthFrame( depthPixels, minDepth, maxDepth, colorPixels, AngleRotateTan, (short)(headDepth - (short)CullingThresh), headPosition); Image<Bgra, byte> depthImg; //Console.WriteLine("iteration:" + sw.ElapsedMilliseconds); sw.Restart(); Image<Gray, Byte> rightFront = null; Image<Gray, Byte> leftFront = null; depthImg = ImageConverter.Array2Image(colorPixels, width, height, width * 4); PointF rightVector = PointF.Empty; PointF leftVector = PointF.Empty; bool isSkip = false; bool leftHandRaise = false; if (skeletons != null && skeletons[0].TrackingState == SkeletonTrackingState.Tracked) { PointF hr = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandRight].Position); PointF hl = SkeletonPointToScreen(skeletons[0].Joints[JointType.HandLeft].Position); PointF er = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowRight].Position); PointF el = SkeletonPointToScreen(skeletons[0].Joints[JointType.ElbowLeft].Position); PointF hip = SkeletonPointToScreen(skeletons[0].Joints[JointType.HipCenter].Position); // hand is lower than hip //Console.WriteLine(skeletons[0].Joints[JointType.HandRight].Position.Y); //Console.WriteLine(skeletons[0].Joints[JointType.HipCenter].Position.Y); //Console.WriteLine("-------------"); if (skeletons[0].Joints[JointType.HandRight].Position.Y < skeletons[0].Joints[JointType.HipCenter].Position.Y + 0.05) { isSkip = true; } if (skeletons[0].Joints[JointType.HandLeft].Position.Y > skeletons[0].Joints[JointType.HipCenter].Position.Y ) { leftHandRaise = true; } rightVector.X = (hr.X - er.X); rightVector.Y = (hr.Y - er.Y); leftVector.X = (hl.X - el.X); leftVector.Y = (hl.Y - el.Y); } HandShapeModel handModel = null; if (!isSkip) { //handModel = m_OpenCVController.FindHandPart(ref depthImg, out rightFront, out leftFront, headDepth - (int)CullingThresh, rightVector, leftVector,leftHandRaise); } // no hands detected if (handModel == null) { handModel = new HandShapeModel(0, HandEnum.None); } //sw.Restart(); // database processing DBManager db = DBManager.GetSingleton(); if (db != null) { if (skeletons != null) { handModel.SetSkeletonData(skeletons[0]); } db.AddFrameData(handModel); } // not recording show prob else { //Image<Bgr, byte>[] result = HandShapeClassifier.GetSingleton() //.RecognizeGesture(handModel.hogRight, 3); ////Console.WriteLine(sw.ElapsedMilliseconds); //if (result != null) //{ // ImageConverter.UpdateWriteBMP(WrtBMP_Candidate1, result[0].Convert<Gray, byte>().ToBitmap()); // ImageConverter.UpdateWriteBMP(WrtBMP_Candidate2, result[1].Convert<Gray, byte>().ToBitmap()); // ImageConverter.UpdateWriteBMP(WrtBMP_Candidate3, result[2].Convert<Gray, byte>().ToBitmap()); //} } string currentSign = db == null ? "0" : db.CurrentSign.ToString(); string path = @"J:\Kinect data\Aaron 141-180\hands\" + currentSign + " " + handModel.frame.ToString(); // UI update if (rightFront != null) { Bitmap right = rightFront.ToBitmap(); //right.Save(path + " r.jpg"); ImageConverter.UpdateWriteBMP(WrtBMP_RightHandFront, right); } if (leftFront != null) { Bitmap left = leftFront.ToBitmap(); //left.Save(path + " l.jpg"); ImageConverter.UpdateWriteBMP(WrtBMP_LeftHandFront, left); } if (sw.ElapsedMilliseconds>15) { Console.WriteLine("Find hand:" + sw.ElapsedMilliseconds); } sw.Restart(); //**************************draw gray histogram //Bitmap histo = m_OpenCVController.Histogram(depthImg); //ImageConverter.UpdateWriteBMP(GrayWriteBitmap, histo); // draw hand position from kinect // DrawHandPosition(depthBMP, rightHandPosition, System.Drawing.Brushes.Purple); //*******************upadte UI ImageConverter.UpdateWriteBMP(DepthWriteBitmap, depthImg.ToBitmap()); // Console.WriteLine("Update UI:" + sw.ElapsedMilliseconds); } } }
public void WriteFrameData(HandShapeModel handModel) { if (connection == null || command == null) { return; } // cmd = String.Format("INSERT INTO FrameData " + // "(FrameDataId, SignInfoId, FrameCount, OffsetedFrameCount, " + //0, 1, 2 //"SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " + //"SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " + //"SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " + //"SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " + //"SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " + //"SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " + //"SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " + //"SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " + //"SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " + //"SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " + //"SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " + //"SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " + //"SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " + //"SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " + // "HandCount, " + // "Hand0FingertipCount, " + // "Hand0Fingertip0X, Hand0Fingertip0Y, Hand0Fingertip0Z, " + // "Hand0Fingertip1X, Hand0Fingertip1Y, Hand0Fingertip1Z, " + // "Hand0Fingertip2X, Hand0Fingertip2Y, Hand0Fingertip2Z, " + // "Hand0Fingertip3X, Hand0Fingertip3Y, Hand0Fingertip3Z, " + // "Hand0Fingertip4X, Hand0Fingertip4Y, Hand0Fingertip4Z, " + // "Hand0EllipseCenterX, Hand0EllipseCenterY, " + // "Hand0EllipseMajorAxis, Hand0EllipseMinorAxis, " + // "Hand0EllipseAspectRatio, Hand0AxisTheta, " + // "Hand1FingertipCount, " + // "Hand1Fingertip0X, Hand1Fingertip0Y, Hand1Fingertip0Z, " + // "Hand1Fingertip1X, Hand1Fingertip1Y, Hand1Fingertip1Z, " + // "Hand1Fingertip2X, Hand1Fingertip2Y, Hand1Fingertip2Z, " + // "Hand1Fingertip3X, Hand1Fingertip3Y, Hand1Fingertip3Z, " + // "Hand1Fingertip4X, Hand1Fingertip4Y, Hand1Fingertip4Z, " + // "Hand1EllipseCenterX, Hand1EllipseCenterY, " + // "Hand1EllipseMajorAxis, Hand1EllipseMinorAxis, " + // "Hand1EllipseAspectRatio, Hand1AxisTheta) " + // "VALUES(NULL, {0}, {1}, {2}{3})", // SignInfoId, frameData.frameCount, frameData.OffsetedFrameCount, frameData.GetFrameDataArgString() // ); // insertions.Add(cmd); //} else { // if (insertions.Count > 0) { //Batch insertion // var sw = Stopwatch.StartNew(); // reader.Close(); // SQLiteTransaction trans = connection.BeginTransaction(); // <------------------- // try // { // foreach (string insertion in insertions) // { // command.CommandText = insertion; // command.ExecuteNonQuery(); // } // insertions.Clear(); // trans.Commit(); // <------------------- // } // catch // { // trans.Rollback(); // <------------------- // throw; // <------------------- // } // Console.WriteLine("save data:" + sw.ElapsedMilliseconds); // } //} //if (!reader.IsClosed) { // reader.Close(); //} ////if (cmd != String.Empty) { //// command.CommandText = cmd; //// command.ExecuteNonQuery(); ////} }
public bool AddFrameData(HandShapeModel hand) { currentFrame++; if (!Begin || hand == null) { return(false); } try { SQLiteCommand insertCommand = new SQLiteCommand(connection); string insertframe = String.Format("INSERT OR REPLACE INTO FrameData (" + "SampleIndex,FrameNumber," + "SkeletonHeadX, SkeletonHeadY, SkeletonHeadZ, " + "SkeletonShoulderCenterX, SkeletonShoulderCenterY, SkeletonShoulderCenterZ, " + "SkeletonShoulderLeftX, SkeletonShoulderLeftY, SkeletonShoulderLeftZ, " + "SkeletonShoulderRightX, SkeletonShoulderRightY, SkeletonShoulderRightZ, " + "SkeletonSpineX, SkeletonSpineY, SkeletonSpineZ, " + "SkeletonHipCenterX, SkeletonHipCenterY, SkeletonHipCenterZ, " + "SkeletonHipLeftX, SkeletonHipLeftY, SkeletonHipLeftZ, " + "SkeletonHipRightX, SkeletonHipRightY, SkeletonHipRightZ, " + "SkeletonElbowLeftX, SkeletonElbowLeftY, SkeletonElbowLeftZ, " + "SkeletonWristLeftX, SkeletonWristLeftY, SkeletonWristLeftZ, " + "SkeletonHandLeftX, SkeletonHandLeftY, SkeletonHandLeftZ, " + "SkeletonElbowRightX, SkeletonElbowRightY, SkeletonElbowRightZ, " + "SkeletonWristRightX, SkeletonWristRightY, SkeletonWristRightZ, " + "SkeletonHandRightX, SkeletonHandRightY, SkeletonHandRightZ, " + "RightHandHOG, LeftHandHOG) VALUES" + "(@SampleID, @Frame" + "{0},@Right,@Left)", hand.skeletonData); insertCommand = new SQLiteCommand(insertframe, connection); insertCommand.Parameters.AddWithValue("@SampleID", CurrentSign); insertCommand.Parameters.AddWithValue("@Frame", currentFrame); if (hand.hogRight == null || hand.hogRight.Length == 0) { insertCommand.Parameters.AddWithValue("@Right", null); } else { insertCommand.Parameters.Add("@Right", DbType.Binary, hand.hogRight.Length * sizeof(float)).Value = hand.hogRight.ToByteArray(); } if (hand.hogLeft == null || hand.hogLeft.Length == 0) { insertCommand.Parameters.AddWithValue("@Left", null); } else { insertCommand.Parameters.Add("@Left", DbType.Binary, hand.hogLeft.Length * sizeof(float)).Value = hand.hogLeft.ToByteArray(); } insertCommand.ExecuteNonQuery(); string sql = @"select last_insert_rowid()"; insertCommand.CommandText = sql; long lastId = (long)insertCommand.ExecuteScalar(); hand.frame = lastId; // update sign type if (hand.type == HandEnum.Intersect) { string update = @"UPDATE signsample set intersected = 1 where index_id = @ID "; SQLiteCommand updateCommand = new SQLiteCommand(update, connection); updateCommand.Parameters.AddWithValue("@ID", CurrentSign); //Console.WriteLine(updateCommand.CommandText); updateCommand.ExecuteNonQuery(); } return(true); } catch (Exception e) { tran.Rollback(); return(false); } }