void RuntimeDepthFrameReady(AllFramesReadyEventArgs e) { using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame == null) { return; } //turn raw data into array of distances var depthArray = depthFrame.ToDepthArray(); //get image DepthImage.Image = depthFrame.ToBitmap(); //get midpoint MidPointDistanceViaGetDistanceText.Text = depthFrame.GetDistance(depthFrame.Width/2, depthFrame.Height/2).ToString(); //image DepthImageWithMinDistance.Image = depthArray.ToBitmap(depthFrame.Width, depthFrame.Height, _minDistance, Color.FromArgb(255, 255, 0, 0)); if (_saveDepthFrame) { _saveDepthFrame = false; depthFrame.ToBitmap().Save(DateTime.Now.ToString("yyyyMMddHHmmss") + "_depth.jpg", ImageFormat.Jpeg); } } }
void kinect_AllFramesReady( object sender, AllFramesReadyEventArgs e ) { // 赤外線画像を表示する using ( ColorImageFrame colorFrame = e.OpenColorImageFrame() ) { if ( colorFrame != null ) { // 赤外線画像データを取得する byte[] color = new byte[colorFrame.PixelDataLength]; colorFrame.CopyPixelDataTo( color ); // 赤外線画像を表示する(16bitのグレースケール) imageInfrared.Source = BitmapSource.Create( colorFrame.Width, colorFrame.Height, 96, 96, PixelFormats.Gray16, null, color, colorFrame.Width * colorFrame.BytesPerPixel ); } } // 距離データを表示する using ( DepthImageFrame depthFrame = e.OpenDepthImageFrame() ) { if ( depthFrame != null ) { // 可視画像に変換する short[] depth = new short[depthFrame.PixelDataLength]; depthFrame.CopyPixelDataTo( depth ); for ( int i = 0; i < depth.Length; i++ ) { depth[i] = (short)~depth[i]; } imageDepth.Source = BitmapSource.Create( depthFrame.Width, depthFrame.Height, 96, 96, PixelFormats.Gray16, null, depth, depthFrame.Width * depthFrame.BytesPerPixel ); } } }
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || _sensor == null) { return; } DepthImagePoint headDepthPoint = this._sensor.CoordinateMapper.MapSkeletonPointToDepthPoint( first.Joints[JointType.Head].Position, DepthImageFormat.Resolution640x480Fps30); DepthImagePoint leftDepthPoint = this._sensor.CoordinateMapper.MapSkeletonPointToDepthPoint( first.Joints[JointType.HandLeft].Position, DepthImageFormat.Resolution640x480Fps30); DepthImagePoint rightDepthPoint = this._sensor.CoordinateMapper.MapSkeletonPointToDepthPoint( first.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30); ColorImagePoint headColorPoint = this._sensor.CoordinateMapper.MapDepthPointToColorPoint( DepthImageFormat.Resolution640x480Fps30, headDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); ColorImagePoint leftColorPoint = this._sensor.CoordinateMapper.MapDepthPointToColorPoint( DepthImageFormat.Resolution640x480Fps30, leftDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); ColorImagePoint rightColorPoint = this._sensor.CoordinateMapper.MapDepthPointToColorPoint( DepthImageFormat.Resolution640x480Fps30, rightDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); CameraPosition(ellipseHead, headColorPoint); CameraPosition(ellipseLeft, leftColorPoint); CameraPosition(ellipseRight, rightColorPoint); } }
void kinect_AllFramesReady( object sender, AllFramesReadyEventArgs e ) { using ( var colorFrame = e.OpenColorImageFrame() ) { if ( colorFrame != null ) { var pixel = new byte[colorFrame.PixelDataLength]; colorFrame.CopyPixelDataTo( pixel ); ImageRgb.Source = BitmapSource.Create( colorFrame.Width, colorFrame.Height, 96, 96, PixelFormats.Bgr32, null, pixel, colorFrame.Width * 4 ); } } using ( var depthFrame = e.OpenDepthImageFrame() ) { if ( depthFrame != null ) { // Depth情報を入れる // GetRawPixelData()はインタラクションライブラリ内で実装された拡張メソッド stream.ProcessDepth( depthFrame.GetRawPixelData(), depthFrame.Timestamp ); } } using ( var skeletonFrame = e.OpenSkeletonFrame() ) { if ( skeletonFrame != null ) { var skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength]; skeletonFrame.CopySkeletonDataTo( skeletons ); // スケルトン情報を入れる stream.ProcessSkeleton( skeletons, kinect.AccelerometerGetCurrentReading(), skeletonFrame.Timestamp ); } } }
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || kinectSensorChooser1.Kinect == null) { return; } //Map a joint location to a point on the depth map //left hand DepthImagePoint leftDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); //Map a depth point to a point on the color image //left hand ColorImagePoint leftColorPoint = depth.MapToColorImagePoint(leftDepthPoint.X, leftDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); CameraPosition(arrow, leftColorPoint); SwipeCheck(leftColorPoint.X); } }
bool working = false; // Skip frames if we're still processing stuff. public void CalibrationAllFramesReady(object sender, AllFramesReadyEventArgs e) { if (working) return; working = true; using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (colorFrame == null) { working = false; return; } using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame == null) { working = false; return; } //byte[] pixels = new byte[colorFrame.PixelDataLength]; //colorFrame.CopyPixelDataTo(pixels); //int stride = colorFrame.Width * 4; //debugImage.Source = BitmapSource.Create(colorFrame.Width, colorFrame.Height, 96, 96, PixelFormats.Bgr32, null, pixels, stride); //debugImage.Visibility = Visibility.Visible; //int code_num = find_code(colorFrame, depthFrame); int code_num = find_touch(colorFrame, depthFrame); if (code_num >= 0) { // Make the next code visible. if (code_num < 4) { codes[code_num].Visibility = Visibility.Hidden; codes[code_num + 1].Visibility = Visibility.Visible; next_code_num++; Thread.Sleep(3000); } else { Thread.Sleep(3000); // We are done. Calculate the coefficients. sensor.AllFramesReady -= this.CalibrationAllFramesReady; codes[4].Visibility = Visibility.Hidden; kinectController.calibration_coefficients = get_calibration_coeffs(); Point center_top_left = code_points[0]; Point center_bot_right = code_points[4]; kinectController.Calibrate((int)(center_top_left.X + 1.25*code_size.X), (int)(center_top_left.Y + 0.7*code_size.Y), (int)(center_bot_right.X - 1.25*code_size.X), (int)(center_bot_right.Y - 0.8*code_size.Y)); sensor.AllFramesReady += kinectController.SensorAllFramesReady; CalibrationDidComplete(); } } } } working = false; }
private void kinect_AllFramesReady(object sender, AllFramesReadyEventArgs e) { byte[] imagem = ObterImagemSensorRGB(e.OpenColorImageFrame()); if (chkEscalaCinza.IsChecked.HasValue && chkEscalaCinza.IsChecked.Value) ReconhecerDistancia(e.OpenDepthImageFrame(), imagem, 2000); if (imagem != null) imagemCamera.Source = BitmapSource.Create(kinect.ColorStream.FrameWidth, kinect.ColorStream.FrameHeight, 96, 96, PixelFormats.Bgr32, null, imagem, kinect.ColorStream.FrameBytesPerPixel * kinect.ColorStream.FrameWidth); }
/// <summary> /// Manage frames of kinect sensor according to the services activated /// </summary> /// <param name="e"></param> private void ManageAllFrame(AllFramesReadyEventArgs e) { if (!IsRunning) { return; } // SkeletonTracking Frame Manager using (SkeletonFrame SFrame = e.OpenSkeletonFrame()) { try { ManageSkeletonFrame(SFrame); } catch (Exception ex) { // Just log the error Console.Error.WriteLine("Error with skeleton frame : " + ex.Message + " _ " + ex.StackTrace); } } // Color Frame Manager if (PropertiesPluginKinect.Instance.EnableColorFrameService) { using (ColorImageFrame CFrame = e.OpenColorImageFrame()) { try { ManageColorFrame(CFrame); } catch (Exception ex) { // Just log the error Console.Error.WriteLine("Error with color frame : " + ex.Message + " _ " + ex.StackTrace); } } } // Depth Frame Manager if (PropertiesPluginKinect.Instance.EnableDepthFrameService || PropertiesPluginKinect.Instance.KinectPointingModeEnabled || PropertiesPluginKinect.Instance.EnableGestureGrip) { using (DepthImageFrame DFrame = e.OpenDepthImageFrame()) { try { ManageDepthFrame(DFrame); } catch (Exception ex) { // Just log the error Console.Error.WriteLine("Error with depth frame : " + ex.Message + " _ " + ex.StackTrace); } } } }
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || _sensor == null) { return; } //Map a joint location to a point on the depth map //head DepthImagePoint headDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); //left hand DepthImagePoint leftHandDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); //right hand DepthImagePoint rightHandDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); //left foot DepthImagePoint leftFootDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); //right foot DepthImagePoint rightFootDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); //hip DepthImagePoint hipDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HipCenter].Position); //Map a depth point to a point on the color image //head ColorImagePoint headColorPoint = depth.MapToColorImagePoint(headDepthPoint.X, headDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left hand ColorImagePoint leftHandColorPoint = depth.MapToColorImagePoint(leftHandDepthPoint.X, leftHandDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right hand ColorImagePoint rightHandColorPoint = depth.MapToColorImagePoint(rightHandDepthPoint.X, rightHandDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left foot ColorImagePoint leftFootColorPoint = depth.MapToColorImagePoint(leftFootDepthPoint.X, leftFootDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right foot ColorImagePoint rightFootColorPoint = depth.MapToColorImagePoint(rightFootDepthPoint.X, rightFootDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //hip ColorImagePoint hipColorPoint = depth.MapToColorImagePoint(hipDepthPoint.X, hipDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); } }
// すべてのデータの更新通知を受け取る void kinect_AllFramesReady(object sender, AllFramesReadyEventArgs e) { using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { RenderScreen(colorFrame, depthFrame); } } }
private void Kinect_AllFramesReady(object sender, AllFramesReadyEventArgs e) { if (cnt++ < 3) return; cnt = 0; using (var f = e.OpenDepthImageFrame()) { var pd = f.GetRawPixelData(); int mx = 1000, my = 1000, Mx = 0, My = 0; for (int i = 0; i < 320; i++) { for (int j = 0; j < 240; j++) { if (get(pd, i, j).PlayerIndex > 0) { if (i < mx) mx = i; if (i > Mx) Mx = i; if (j < my) my = j; if (j > My) My = j; } } } if (mx<1000) { float wc = (Mx - mx) / 16; float hc = (My - my) / 16; if (keep_proportions) { if (hc < wc) hc = wc; else wc = hc; } Console.WriteLine("mx={0},Mx={1},my={2},My={3}", mx, Mx, my, My); for (int j=0;j<16;j++) { int b1 = 0; for (int i = 0; i<8; i++) { b1 = b1 * 2 + ((get(pd, mx + (int)(wc * i), my + (int)(hc * j)).PlayerIndex > 0) ? 1 : 0); } int b2 = 0; for (int i = 8; i < 16; i++) { b2 = b2 * 2 + ((get(pd, mx + (int)(wc * i), my + (int)(hc * j)).PlayerIndex > 0) ? 1 : 0); } byte[] x = new byte[2]; x[0] = (byte)b1; x[1] = (byte)b2; COM.Write(x, 0, 2); Console.WriteLine("Sending {0},{1}", b1, b2); } } // Console.Write("{0}\r", pd.Length); } }
private void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame == null || _sensor == null) return; // DepthImagePoint headDepthPoint = depthFrame.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); //left hand point information DepthImagePoint LeftHandDepthPoint = depthFrame.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); LeftHandPoint newLeftHandPoint = new LeftHandPoint() { X = LeftHandDepthPoint.X, Y = LeftHandDepthPoint.Y, Z = LeftHandDepthPoint.Depth, T = DateTime.Now }; DepthImagePoint HeadDepthPoint = depthFrame.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); HeadPoint newHeadPoint = new HeadPoint() { X = HeadDepthPoint.X, Y = HeadDepthPoint.Y, Z = HeadDepthPoint.Depth, T = DateTime.Now }; //user should stand in the right place before eveything start // the two if condition requires the user to stand in front of Kinect in a box area if (newHeadPoint.Z < 1700 || newHeadPoint.Z > 2000) { StatusLabel.Visibility = System.Windows.Visibility.Hidden; StatusLabel.Content = ""; return; } StatusLabel.Visibility = System.Windows.Visibility.Visible; StatusLabel.Content = "Control Mode(1.7m~2m): " + newHeadPoint.Z / 1000 + "m"; // left hand wave to quit system if (newLeftHandPoint.Y < newHeadPoint.Y) { // MessageBox.Show("Left wave"); LeftHandWave(newLeftHandPoint, newHeadPoint); } else { IsLeftHandWave = false; } }// end of using statement }
private void Kinect_AllFramesReady(object sender, AllFramesReadyEventArgs allFrameEvent) { byte[] imagem = ObterImagemSensorRGB(allFrameEvent.OpenColorImageFrame()); FuncoesProfundidade(allFrameEvent.OpenDepthImageFrame(), imagem, 2000); if (imagem != null) { canvasKinect.Background = new ImageBrush(BitmapSource.Create(Kinect.ColorStream.FrameWidth, Kinect.ColorStream.FrameHeight, 96, 96, PixelFormats.Bgr32, null, imagem, Kinect.ColorStream.FrameBytesPerPixel * Kinect.ColorStream.FrameWidth)); } canvasKinect.Children.Clear(); FuncoesEsqueletoUsuario(allFrameEvent.OpenSkeletonFrame()); }
void newSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame == null) { return; } byte[] pixels = GenerateColoredBytes(depthFrame); int stride = depthFrame.Width * 4; image1.Source = BitmapSource.Create(depthFrame.Width, depthFrame.Height, 96, 96, PixelFormats.Bgr32, null, pixels, stride); } }
private void kinect_AllFramesReady(object sender, AllFramesReadyEventArgs e) { byte[] imagem = ObterImagemSensorRGB(e.OpenColorImageFrame()); if( chkEscalaCinza.IsChecked.HasValue && chkEscalaCinza.IsChecked.Value) ReconhecerDistancia(e.OpenDepthImageFrame(),imagem, 2000); if (imagem != null) canvasKinect.Background = new ImageBrush(BitmapSource.Create(kinect.ColorStream.FrameWidth, kinect.ColorStream.FrameHeight, 96, 96, PixelFormats.Bgr32, null, imagem, kinect.ColorStream.FrameWidth * kinect.ColorStream.FrameBytesPerPixel)); canvasKinect.Children.Clear(); DesenharEsqueletoUsuario(e.OpenSkeletonFrame()); }
// すべてのデータの更新通知を受け取る void kinect_AllFramesReady( object sender, AllFramesReadyEventArgs e ) { // Disposableなのでusingでくくる using ( ColorImageFrame colorFrame = e.OpenColorImageFrame() ) { if ( colorFrame != null ) { imageRgbCamera.Source = colorFrame.ToBitmapSource(); } } // Disposableなのでusingでくくる using ( DepthImageFrame depthFrame = e.OpenDepthImageFrame() ) { if ( depthFrame != null ) { imageDepthCamera.Source = depthFrame.ToBitmapSource(); } } }
/// <summary> /// Handles the Kinect AllFramesReady event /// </summary> private void Sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { ColorImageFrame colorFrame = null; DepthImageFrame depthFrame = null; Skeleton[] skeletonData; try { colorFrame = e.OpenColorImageFrame(); depthFrame = e.OpenDepthImageFrame(); using (var skeletonFrame = e.OpenSkeletonFrame()) { if (colorFrame == null || depthFrame == null || skeletonFrame == null) return; skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; skeletonFrame.CopySkeletonDataTo(skeletonData); } // Find a skeleton to track. // First see if our old one is good. // When a skeleton is in PositionOnly tracking state, don't pick a new one // as it may become fully tracked again. Skeleton skeletonOfInterest = skeletonData.FirstOrDefault(s => s.TrackingId == this.trackedSkeletonId && s.TrackingState != SkeletonTrackingState.NotTracked); if (skeletonOfInterest == null) { // Old one wasn't around. Find any skeleton that is being tracked and use it. skeletonOfInterest = skeletonData.FirstOrDefault(s => s.TrackingState == SkeletonTrackingState.Tracked); if (skeletonOfInterest != null) this.trackedSkeletonId = skeletonOfInterest.TrackingId; } if (this.FrameDataUpdated != null) this.FrameDataUpdated(this, new FrameData(colorFrame, depthFrame, skeletonOfInterest)); } finally { if (colorFrame != null) colorFrame.Dispose(); if (depthFrame != null) depthFrame.Dispose(); } }
void GetCameraPoint(Skeleton first, bool flag, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || kinectSensorChooser1.Kinect == null) { return; } //Map a joint location to a point on the depth map //head DepthImagePoint headDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); //left hand DepthImagePoint leftDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); //right hand DepthImagePoint rightDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); //Map a depth point to a point on the color image //head ColorImagePoint headColorPoint = depth.MapToColorImagePoint(headDepthPoint.X, headDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left hand ColorImagePoint leftColorPoint = depth.MapToColorImagePoint(leftDepthPoint.X, leftDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right hand ColorImagePoint rightColorPoint = depth.MapToColorImagePoint(rightDepthPoint.X, rightDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30 ); //Set location if(flag) CameraPosition(headImage, headColorPoint); else { CameraPosition(headImage2, headColorPoint); } CameraPosition(leftEllipse, leftColorPoint); CameraPosition(rightEllipse, rightColorPoint); } }
public void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || KinectSensor.Kinect == null) { return; } //Map a joint location to a point on the depth map //head DepthImagePoint headDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); //left hand DepthImagePoint leftDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); //right hand DepthImagePoint rightDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); //Map a depth point to a point on the color image //head ColorImagePoint headColorPoint = depth.MapToColorImagePoint(headDepthPoint.X, headDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left hand ColorImagePoint leftColorPoint = depth.MapToColorImagePoint(leftDepthPoint.X, leftDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right hand ColorImagePoint rightColorPoint = depth.MapToColorImagePoint(rightDepthPoint.X, rightDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //Set location //CameraPosition(Head, headColorPoint); //CameraPosition(LeftHand, leftColorPoint); //CameraPosition(RightHand, rightColorPoint); } }
public void SensorAllFramesReady(object sender, AllFramesReadyEventArgs e) { bool depthReceived = false; bool colorReceived = false; using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (null != depthFrame) { // Copy the pixel data from the image to a temporary array depthFrame.CopyDepthImagePixelDataTo(depthPixels); depthReceived = true; } } using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (null != colorFrame) { // Done by WSRKinectMacro colorReceived = true; } } if (depthReceived) { HandleDepth(); } if (colorReceived) { HandleColor(); } WSRKinect wsr = (WSRKinect)WSRConfig.GetInstance().GetWSRMicro(); WriteableBitmap resize = colorBitmap.Resize(320, 240, WriteableBitmapExtensions.Interpolation.Bilinear); Bitmap image = wsr.GetColorPNG(resize, false); MemoryStream ms = new MemoryStream(); image.Save(ms, format); image.Dispose(); byte[] imgByte = ms.ToArray(); base64String = Convert.ToBase64String(imgByte); // lock (sockets) { // foreach (var socket in sockets) { SendWebSocket(socket, image); } // image.Dispose(); // } }
public void SensorAllFramesReady(object sender, AllFramesReadyEventArgs e) { // Check if we need to change background if (drawController.backgroundAlreadySet == false) { drawController.ChangeBackground(drawController.background); } // Check if we need to change color if (drawController.shouldChangeColor != -1) { drawController.ChangeColor((Colors)drawController.shouldChangeColor); } using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame != null) { this.ParseDepthFrame(depthFrame); } } }
public void DisplayColorImageAllFramesReady(object sender, AllFramesReadyEventArgs e) { using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { //Console.WriteLine("here"); if (colorFrame == null) return; byte[] pixels = new byte[colorFrame.PixelDataLength]; colorFrame.CopyPixelDataTo(pixels); int stride = colorFrame.Width * 4; debugImage.Source = BitmapSource.Create(colorFrame.Width, colorFrame.Height, 96, 96, PixelFormats.Bgr32, null, pixels, stride); using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { short[] rawDepthData = new short[depthFrame.PixelDataLength]; depthFrame.CopyPixelDataTo(rawDepthData); int depth = rawDepthData[200*depthFrame.Width + 300] >> DepthImageFrame.PlayerIndexBitmaskWidth; threshold = depth; } } }
void getCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { // Gets the data from the Kinect and sends it to the sendData method using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || kinectSensorChooser1.Kinect == null) { return; } DepthImagePoint headDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); DepthImagePoint leftDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); DepthImagePoint rightDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); ColorImagePoint headColorPoint = depth.MapToColorImagePoint(headDepthPoint.X, headDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); ColorImagePoint leftColorPoint = depth.MapToColorImagePoint(leftDepthPoint.X, leftDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); ColorImagePoint rightColorPoint = depth.MapToColorImagePoint(rightDepthPoint.X, rightDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); CameraPosition(leftHand, leftColorPoint); CameraPosition(rightHand, rightColorPoint); sendData(rightColorPoint.X, rightColorPoint.Y); } }
void kinect_AllFramesReady(object sender, AllFramesReadyEventArgs e) { // Color using (var frame = e.OpenColorImageFrame()) { //Get raw image if (frame != null) { //Create array for pixel data and copy it from the image frame Byte[] pixelData = new Byte[frame.PixelDataLength]; frame.CopyPixelDataTo(pixelData); //Convert RGBA to BGRA Byte[] bgraPixelData = new Byte[frame.PixelDataLength]; for (int i = 0; i < pixelData.Length; i += 4) { bgraPixelData[i] = pixelData[i + 2]; bgraPixelData[i + 1] = pixelData[i + 1]; bgraPixelData[i + 2] = pixelData[i]; bgraPixelData[i + 3] = (Byte)255; //The video comes with 0 alpha so it is transparent } // Create a texture and assign the realigned pixels colorVideo = new Texture2D(game.GraphicsDevice, frame.Width, frame.Height); colorVideo.SetData(bgraPixelData); } } // Depth using (var frame = e.OpenDepthImageFrame()) { if (frame != null) { short[] pixelData = new short[frame.PixelDataLength]; pixelData = new short[frame.PixelDataLength]; frame.CopyPixelDataTo(pixelData); depthVideo = new Texture2D(game.GraphicsDevice, frame.Width, frame.Height); depthVideo.SetData(ConvertDepthFrame(pixelData, kinectSensor.DepthStream)); } } using (var frame = e.OpenSkeletonFrame()) { if (frame != null) { skeletonPoints = new List <ColoredPoint>(); frame.CopySkeletonDataTo(this.skeletons); //refresh the closest skeletons this.TrackClosestSkeleton(); foreach (var body in skeletons) { if (body != null && body.TrackingState == SkeletonTrackingState.Tracked) { foreach (Joint joint in body.Joints) { SkeletonPoint skeletonPoint = joint.Position; // 2D coordinates in pixels Point point = new Point(); if (drawState == KinectDraw.Camera) { // Skeleton-to-Color mapping ColorImagePoint colorPoint = kinectSensor.CoordinateMapper.MapSkeletonPointToColorPoint(skeletonPoint, ColorImageFormat.RgbResolution640x480Fps30); point.X = colorPoint.X; point.Y = colorPoint.Y; } if (drawState == KinectDraw.Depth) // Remember to change the Image and Canvas size to 320x240. { // Skeleton-to-Depth mapping DepthImagePoint depthPoint = kinectSensor.CoordinateMapper.MapSkeletonPointToDepthPoint(skeletonPoint, DepthImageFormat.Resolution640x480Fps30); point.X = depthPoint.X; point.Y = depthPoint.Y; } Color colorToAdd; if (trackedSkeletonOne != null) { if (body.TrackingId == trackedSkeletonOne.TrackingId) { colorToAdd = Color.Red; } else if (body.TrackingId == trackedSkeletonTwo.TrackingId) { colorToAdd = Color.Blue; } else { //something obnoxious colorToAdd = Color.HotPink; } } else { //something obnoxious colorToAdd = Color.HotPink; } skeletonPoints.Add(new ColoredPoint(point, colorToAdd)); } } } } } }
void myKinect_AllFramesReady(object sender, AllFramesReadyEventArgs e) { #if SREENSHOT_PAUSE if (trackCount == trackLimit) { return; } #endif #region Video image // Puts a copy of the video image into the kinect video texture using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (colorFrame == null) { return; } if (colorData == null) { colorData = new byte[colorFrame.Width * colorFrame.Height * 4]; } colorFrame.CopyPixelDataTo(colorData); kinectVideoTexture = new Texture2D(GraphicsDevice, colorFrame.Width, colorFrame.Height); Color[] bitmap = new Color[colorFrame.Width * colorFrame.Height]; int sourceOffset = 0; for (int i = 0; i < bitmap.Length; i++) { bitmap[i] = new Color(colorData[sourceOffset + 2], colorData[sourceOffset + 1], colorData[sourceOffset], 255); sourceOffset += 4; } kinectVideoTexture.SetData(bitmap); } #endregion #region Skeleton // Finds the currently active skeleton using (SkeletonFrame frame = e.OpenSkeletonFrame()) { if (frame == null) { return; } else { skeletons = new Skeleton[frame.SkeletonArrayLength]; frame.CopySkeletonDataTo(skeletons); } } activeSkeletonNumber = 0; for (int i = 0; i < skeletons.Length; i++) { if (skeletons[i].TrackingState == SkeletonTrackingState.Tracked) { activeSkeletonNumber = i + 1; activeSkeleton = skeletons[i]; break; } } #endregion #region Depth image // Creates a game background image with transparent regions // where the player is displayed using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { // Get the depth data if (depthFrame == null) { return; } if (depthData == null) { depthData = new short[depthFrame.Width * depthFrame.Height]; } depthFrame.CopyPixelDataTo(depthData); // Create the mask from the background image gameImageTexture.GetData(maskImageColors); if (activeSkeletonNumber != 0) { for (int depthPos = 0; depthPos < depthData.Length; depthPos++) { // find a player to mask - split off bottom bits int playerNo = depthData[depthPos] & 0x07; if (playerNo == activeSkeletonNumber) { // We have a player to mask // find the X and Y positions of the depth point int x = depthPos % depthFrame.Width; int y = depthPos / depthFrame.Width; // get the X and Y positions in the video feed ColorImagePoint playerPoint = myKinect.MapDepthToColorImagePoint( DepthImageFormat.Resolution320x240Fps30, x, y, depthData[depthPos], ColorImageFormat.RgbResolution640x480Fps30); // Map the player coordinates into our lower resolution background // Have to do this because the lowest resultion for the color camera is 640x480 playerPoint.X /= 2; playerPoint.Y /= 2; // convert this into an offset into the mask color data int gameImagePos = (playerPoint.X + (playerPoint.Y * depthFrame.Width)); if (gameImagePos < maskImageColors.Length) { // make this point in the mask transparent maskImageColors[gameImagePos] = Color.FromNonPremultiplied(0, 0, 0, 0); } } } } gameMaskTexture = new Texture2D(GraphicsDevice, depthFrame.Width, depthFrame.Height); gameMaskTexture.SetData(maskImageColors); } #endregion }
private void DecodeSkeletonData(AllFramesReadyEventArgs e, KinectSensor sensor) { #region GetImageFormat ColorImageFormat colorFormat = ColorImageFormat.Undefined; int colorWidth = 0; int colorHeight = 0; DepthImageFormat depthFormat = DepthImageFormat.Undefined; int depthWidth = 0; int depthHeight = 0; switch (this._imageType) { case ImageType.Color: // Retrieve the current color format, from the frame if present, and from the sensor if not. using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame()) { if (null != colorImageFrame) { colorFormat = colorImageFrame.Format; colorWidth = colorImageFrame.Width; colorHeight = colorImageFrame.Height; } else if (null != sensor.ColorStream) { colorFormat = sensor.ColorStream.Format; colorWidth = sensor.ColorStream.FrameWidth; colorHeight = sensor.ColorStream.FrameHeight; } } break; case ImageType.Depth: // Retrieve the current depth format, from the frame if present, and from the sensor if not. using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { if (null != depthImageFrame) { depthFormat = depthImageFrame.Format; depthWidth = depthImageFrame.Width; depthHeight = depthImageFrame.Height; } else if (null != sensor.DepthStream) { depthFormat = sensor.DepthStream.Format; depthWidth = sensor.DepthStream.FrameWidth; depthHeight = sensor.DepthStream.FrameHeight; } } break; } #endregion // Clear the play canvas this.playField.Children.Clear(); // Check every skeleton for (int skeletonSlot = 0; skeletonSlot < this._skeletonBuffer.Length; skeletonSlot++) { var skeleton = this._skeletonBuffer[skeletonSlot]; #region Skeleton Position // Map points between skeleton and color/depth var jointMapping = this._jointMappings[skeletonSlot]; jointMapping.Clear(); try { // Transform the data into the correct space // For each joint, we determine the exact X/Y coordinates for the target view foreach (Joint joint in skeleton.Joints) { ColorImagePoint colorPoint = sensor.CoordinateMapper.MapSkeletonPointToColorPoint(joint.Position, colorFormat); Point mappedPoint = new Point( (int)(this._renderSize.Width * colorPoint.X / colorWidth), (int)(this._renderSize.Height * colorPoint.Y / colorHeight)); jointMapping[joint.JointType] = new JointMapping { Joint = joint, MappedPoint = mappedPoint, OriginPoint = colorPoint, }; } } catch (UnauthorizedAccessException) { // Kinect is no longer available. return; } // Look up the center point Point centerPoint = PositionCalculator.Get2DPosition( sensor, this._imageType, this._renderSize, skeleton.Position, colorFormat, colorWidth, colorHeight, depthFormat, depthWidth, depthHeight); #endregion // Scale the skeleton thickness // 1.0 is the desired size at 640 width this._scaleFactor = this._renderSize.Width / colorWidth; // Displays a gradient near the edge of the display // where the skeleton is leaving the screen this.DrawClippedEdges(skeleton); switch (skeleton.TrackingState) { case SkeletonTrackingState.PositionOnly: { // The skeleton is being tracked, but we only know the general position, and // we do not know the specific joint locations. this.DrawBodyCenter(centerPoint); } break; case SkeletonTrackingState.Tracked: { // The skeleton is being tracked and the joint data is available for consumption. this.DrawBody(skeleton, jointMapping); // Track player this.TrackPlayer(skeleton, jointMapping); } break; } } }
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; File.AppendAllText("mouseLog.txt", DateTime.Now + " - All Kinect frames ready.\n"); try { colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { File.AppendAllText("mouseLog.txt", DateTime.Now + " - Color- depth or Skeletonframe is null. Aborting Frame.\n"); return; } // Check for image format changes. The FaceTracker doesn't // deal with that so we need to reset. if (this.depthImageFormat != depthImageFrame.Format) { this.depthImage = null; this.depthImageFormat = depthImageFrame.Format; } if (this.colorImageFormat != colorImageFrame.Format) { this.colorImage = null; this.colorImageFormat = colorImageFrame.Format; } // Create any buffers to store copies of the data we work with if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } // Get the skeleton information if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } colorImageFrame.CopyPixelDataTo(this.colorImage); depthImageFrame.CopyPixelDataTo(this.depthImage); skeletonFrame.CopySkeletonDataTo(this.SkeletonData); Skeleton activeSkeleton = null; activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault(); //Idea: Separate Eye-Parts of Color Image //Use learning Algorithm for right and left eye //Detect blink on separated parts of color Image //colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values if (activeSkeleton != null) { File.AppendAllText("mouseLog.txt", DateTime.Now + " - Skeleton is there. Trying to find face.\n"); FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton); if (currentFaceFrame.TrackSuccessful) { File.AppendAllText("mouseLog.txt", DateTime.Now + " - Recognized face successfully.\n"); } else { File.AppendAllText("mouseLog.txt", DateTime.Now + " - Couldn't find face in frame.\n"); } //Get relevant Points for blink detection //Left eye int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X); int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y); int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X); int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y); Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false); //this.pbRight.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye)); // //Right eye minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X); minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y); maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X); maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y); Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true); Bitmap leftEye2 = null; Bitmap rightEye2 = null; if (leftEye != null) { leftEye2 = new Bitmap(leftEye); } if (rightEye != null) { rightEye2 = new Bitmap(rightEye); } // System.Delegate d = new MethodInvoker(SetPictures)); // this.Invoke(SetPictures, leftEye); //pbRight.Image = rightEye; this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbLeft.Image = rightEye2)); this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye2)); // this.Invoke(new MethodInvoker(SetPictures)); //Wende Kantenfilter auf die beiden Augen an. if (rightEye != null && leftEye != null) { Dictionary <string, int> angleCount; Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out angleCount); bool rightEyeClosed = IsEyeClosed(angleCount); Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out angleCount); bool leftEyeClosed = IsEyeClosed(angleCount); // pbLeftFaltung.Image = edgePicLeft; // pbRightFaltung.Image = edgePicRight; if (rightEyeClosedHistory.Count > 100) { rightEyeClosedHistory.RemoveAt(0); } if (leftEyeClosedHistory.Count > 100) { leftEyeClosedHistory.RemoveAt(0); } leftEyeClosedHistory.Add(leftEyeClosed); rightEyeClosedHistory.Add(rightEyeClosed); //If Face is rotated, move Mouse if (headRotationHistory.Count > gaussFilter.Count - 1 && leftEyeClosedHistory.Count > nudConvolutionFilterLength.Value && currentFaceFrame.TrackSuccessful) { int x = 0; int y = 0; float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser]; float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower]; float mouthOpenValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.JawLower]; if (browRaiserHistory.Count >= 100) { browRaiserHistory.RemoveAt(0); browLowererHistory.RemoveAt(0); mouthOpenHistory.RemoveAt(0); } browLowererHistory.Add(browLowererValue); browRaiserHistory.Add(browRaiserValue); mouthOpenHistory.Add(mouthOpenValue); //Method 1: Ohne Glättung //ScaleXY(currentFaceFrame.Rotation, out x, out y); //MouseControl.Move(x, y); ////Method 2: Glättung über die letzten x Bilder: //int i = 0; //Vector3DF rotationMedium = new Vector3DF(); //while (i < 10 && headRotationHistory.Count - 1 > i) //{ // i++; // rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X; // rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y; //} //rotationMedium.X = rotationMedium.X / i; //rotationMedium.Y = rotationMedium.Y / i; //ScaleXY(rotationMedium, out x, out y); //MouseControl.Move(x, y); //Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker. Vector3DF rotationMedium = new Vector3DF(); rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0]; rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0]; int i = 0; while (i < gaussFilter.Count - 1) { rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]); rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]); i++; } rotationMedium.X = (float)(rotationMedium.X / gaussFactor); rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor); ScaleXY(rotationMedium, out x, out y); //Method 4: Quadratische Glättung //double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y)); //double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X)); //if (deltaX < 0) // deltaX = -Math.Pow(deltaX, 2) * 4; //else // deltaX = Math.Pow(deltaX, 2) * 4; //if (deltaY < 0) // deltaY = -Math.Pow(deltaY, 2) * 5; //else // deltaY = Math.Pow(deltaY, 2) * 5; //MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY)); //Check for right, left or Double Click //1. Check if there was already a click 20 Frames ago, or if Drag & Drop is active if (clickDelay > nudClickDelay.Value && !pointNClickActive) { //2. If not, calculate mean values of dy's last 16 Frames if (CalculateMeanConvolutionValues()) { clickDelay = 0; } else { //Else check for open Mouth if (mouthOpenValue > (float)nudMouthOpenStartThreshold.Value && mouthOpenHistory[mouthOpenHistory.Count - 2] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 3] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 4] > (float)nudMouthOpenConfirmation.Value) { MouseControl.Move(mousePositionHistory[mousePositionHistory.Count - 4].X, mousePositionHistory[mousePositionHistory.Count - 4].Y); this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y))); //lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y); MouseControl.MouseDownLeft(); pointNClickActive = true; clickDelay = 0; } } } else if (pointNClickActive) { if (mouthOpenValue < (float)nudMouthOpenEndThreshold.Value) { this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Up on X: " + x + " Y: " + y))); MouseControl.MouseUpLeft(); pointNClickActive = false; clickDelay = 0; } } MouseControl.Move(x, y); if (browLowererValue > (float)nudBrowLowererStartThreshold.Value) { MouseControl.ScrollDown((int)(-browLowererValue * (int)nudScrollMultiplierDown.Value)); } if (browRaiserValue > (float)nudBrowRaiserStartThreshold.Value) { MouseControl.ScrollDown((int)(browRaiserValue * (int)nudScrollMultiplierUp.Value)); } if (mousePositionHistory.Count > 100) { mousePositionHistory.RemoveAt(0); } mousePositionHistory.Add(new Microsoft.Kinect.Toolkit.FaceTracking.Point(x, y)); File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face and eyes successfully tracked.\n"); } } else { File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face recognized but couldn't find eye in face.\n"); } clickDelay++; headRotationHistory.Add(currentFaceFrame.Rotation); if (headRotationHistory.Count >= 100) { headRotationHistory.RemoveAt(0); } } else { File.AppendAllText("mouseLog.txt", DateTime.Now + " - Active Skeleton is null. Couldn't analyze frame.\n"); } } catch (Exception e) { File.AppendAllText("mouseLog.txt", DateTime.Now + " - Error during frame analyzation.\n" + e.ToString()); } finally { if (colorImageFrame != null) { colorImageFrame.Dispose(); } if (depthImageFrame != null) { depthImageFrame.Dispose(); } if (skeletonFrame != null) { skeletonFrame.Dispose(); } } }
void sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame == null) { return; } int minDepth = 850; int maxDepth = 4000; System.Drawing.Bitmap outBmp = new System.Drawing.Bitmap(160, 160); BitmapSource depthBitmapSource; BitmapSource processedBitmapSource; //Get the position of interest on the depthmap from skeletal tracking DepthImagePoint rightHandPoint = jointTracker.GetJointPosition(kinectSensorChooser.Kinect, e, JointType.HandRight); if (jointTracker.JointDetected == true) { textResult.Text = "Right hand is being tracked"; int rightHandDepth = rightHandPoint.Depth; if (rightHandDepth < 850) { minDepth = 850; maxDepth = 1500; } else { minDepth = rightHandDepth - 75; maxDepth = rightHandDepth + 75; } depthBitmapSource = sliceDepthImage(depthFrame, minDepth, maxDepth); //Create a bitmap from the depth information System.Drawing.Bitmap depthBmp = depthBitmapSource.ToBitmap(); //Aforge performs image processing here. outBmp = imageProcessor.ProcessFrame(depthBmp, rightHandPoint.X, rightHandPoint.Y); } else { textResult.Text = "No hand detected"; //depthBitmapSource = sliceDepthImage(depthFrame, 850, 1500); System.Drawing.Graphics g = System.Drawing.Graphics.FromImage(outBmp); g.Clear(System.Drawing.Color.Black); } //Create a bitmapsource to show the processed image processedBitmapSource = outBmp.ToBitmapSource(); //Display the images procImageDisplay.Source = processedBitmapSource; } }
private void KinectSensorOnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { if (++fpsGlobal < FPSGlobal) { return; } fpsGlobal = 0; using (var depthFrame = allFramesReadyEventArgs.OpenDepthImageFrame()) using (var colorFrame = allFramesReadyEventArgs.OpenColorImageFrame()) using (var skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame()) { if (null == depthFrame || null == colorFrame || null == skeletonFrame) { return; } AllFrameWatch.Again(); // Init ONCE with provided data InitFrames(depthFrame, colorFrame, skeletonFrame); if (!init) { return; } // Backup frames (motion) depthFrame.CopyPixelDataTo(Depth.Pixelss); // Motion check if (Task == null || Task.StandBy) { AllFrameWatch.Stop(); return; } // Copy computed depth if (++fpsDepth >= FPSDepth) { fpsDepth = 0; // depthFrame.CopyDepthImagePixelDataTo(this.DepthPixels); Depth.Stamp.Time = System.DateTime.Now; } // Copy color data if (++fpsColor >= FPSColor) { fpsColor = 0; colorFrame.CopyPixelDataTo(Color.Pixels); Color.Stamp.Time = System.DateTime.Now; } // Copy skeleton data if (++fpsSkeleton >= FPSSkeleton) { fpsSkeleton = 0; skeletonFrame.CopySkeletonDataTo((Skeleton[])Skeletons.RawData); Skeletons.Stamp.Time = System.DateTime.Now; } // Convert Joint 3D to 2D on 1st skeleton if (++fpsJoints >= FPSJoints) { fpsJoints = 0; RefreshBodyData(Skeletons); } AllFrameWatch.Stop(); } }
void nui_AllFramesReady(object sender, AllFramesReadyEventArgs e) { SkeletonFrame sf = e.OpenSkeletonFrame(); if (sf == null) { return; } Skeleton[] skeletonData = new Skeleton[sf.SkeletonArrayLength]; sf.CopySkeletonDataTo(skeletonData); using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { if (depthImageFrame != null) { foreach (Skeleton sd in skeletonData) { if (sd.TrackingState == SkeletonTrackingState.Tracked) { int nMax = 20; Joint[] joints = new Joint[nMax]; for (int j = 0; j < nMax; j++) { joints[j] = sd.Joints[(JointType)j]; } Point[] points = new Point[nMax]; for (int j = 0; j < nMax; j++) { DepthImagePoint depthPoint; depthPoint = depthImageFrame.MapFromSkeletonPoint(joints[j].Position); points[j] = new Point((int)(image1.Width * depthPoint.X / depthImageFrame.Width), (int)(image1.Height * depthPoint.Y / depthImageFrame.Height)); } PointCollection pc0 = new PointCollection(4); pc0.Add(points[(int)JointType.HipCenter]); pc0.Add(points[(int)JointType.Spine]); pc0.Add(points[(int)JointType.ShoulderCenter]); pc0.Add(points[(int)JointType.Head]); m_poly[0].Points = pc0; m_poly[0].Visibility = Visibility.Visible; //엉덩이부터 머리까지 연결되는 선 PointCollection pc1 = new PointCollection(5); pc1.Add(points[(int)JointType.ShoulderCenter]); pc1.Add(points[(int)JointType.ShoulderLeft]); pc1.Add(points[(int)JointType.ElbowLeft]); pc1.Add(points[(int)JointType.WristLeft]); pc1.Add(points[(int)JointType.HandLeft]); m_poly[1].Points = pc1; m_poly[1].Visibility = Visibility.Visible; //왼손부터 어깨까지 연결되는 선 PointCollection pc2 = new PointCollection(5); pc2.Add(points[(int)JointType.ShoulderCenter]); pc2.Add(points[(int)JointType.ShoulderRight]); pc2.Add(points[(int)JointType.ElbowRight]); pc2.Add(points[(int)JointType.WristRight]); pc2.Add(points[(int)JointType.HandRight]); m_poly[2].Points = pc2; m_poly[2].Visibility = Visibility.Visible; // 오른손부터 어깨까지 연결되는 선 PointCollection pc3 = new PointCollection(5); pc3.Add(points[(int)JointType.HipCenter]); pc3.Add(points[(int)JointType.HipLeft]); pc3.Add(points[(int)JointType.KneeLeft]); pc3.Add(points[(int)JointType.AnkleLeft]); pc3.Add(points[(int)JointType.FootLeft]); m_poly[3].Points = pc3; m_poly[3].Visibility = Visibility.Visible; // 왼발부터 엉덩이까지 연결되는 선 PointCollection pc4 = new PointCollection(5); pc4.Add(points[(int)JointType.HipCenter]); pc4.Add(points[(int)JointType.HipRight]); pc4.Add(points[(int)JointType.KneeRight]); pc4.Add(points[(int)JointType.AnkleRight]); pc4.Add(points[(int)JointType.FootRight]); m_poly[4].Points = pc4; m_poly[4].Visibility = Visibility.Visible; // 오른발부터 엉덩이까지 연결되는 선 } } } } }
private void KinectAllFramesReady(object sender, AllFramesReadyEventArgs e) { // Have we already been "shut down" by the user of this viewer, // or has the SkeletonStream been disabled since this event was posted? if ((this.Kinect == null) || !((KinectSensor)sender).SkeletonStream.IsEnabled) { return; } bool haveSkeletonData = false; using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { if (skeletonFrame != null) { if (this.skeletonCanvases == null) { this.CreateListOfSkeletonCanvases(); } if ((this.skeletonData == null) || (this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } skeletonFrame.CopySkeletonDataTo(this.skeletonData); haveSkeletonData = true; } } if (haveSkeletonData) { using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { if (depthImageFrame != null) { int trackedSkeletons = 0; foreach (Skeleton skeleton in this.skeletonData) { Dictionary <JointType, JointMapping> jointMapping = this.jointMappings[trackedSkeletons]; jointMapping.Clear(); KinectSkeleton skeletonCanvas = this.skeletonCanvases[trackedSkeletons++]; skeletonCanvas.ShowBones = this.ShowBones; skeletonCanvas.ShowJoints = this.ShowJoints; skeletonCanvas.ShowCenter = this.ShowCenter; //Burası // Transform the data into the correct space // For each joint, we determine the exact X/Y coordinates for the target view foreach (Joint joint in skeleton.Joints) { //TODO: if (joint.JointType == JointType.Head) { HeadX = joint.Position.X; HeadY = joint.Position.Y; HeadZ = joint.Position.Z; } if (joint.JointType == JointType.HandRight) { if (joint.Position.X * joint.Position.Y != 0) { RightHandX = joint.Position.X; RightHandY = joint.Position.Y; } } if (joint.JointType == JointType.HandLeft) { if (joint.Position.X * joint.Position.Y != 0) { LeftHandX = joint.Position.X; LeftHandY = joint.Position.Y; } } Point mappedPoint = this.GetPosition2DLocation(depthImageFrame, joint.Position); jointMapping[joint.JointType] = new JointMapping { Joint = joint, MappedPoint = mappedPoint }; } // Look up the center point Point centerPoint = this.GetPosition2DLocation(depthImageFrame, skeleton.Position); // Scale the skeleton thickness // 1.0 is the desired size at 640 width double scale = this.RenderSize.Width / 640; skeletonCanvas.RefreshSkeleton(skeleton, jointMapping, centerPoint, scale); } //TODO: FrameReloaded(); if (ImageType == ImageType.Depth) { this.ChooseTrackedSkeletons(this.skeletonData); } } } } }
void Sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { // Color using (var frame = e.OpenColorImageFrame()) { if (frame != null) { if (_mode == CameraMode.Color) { camera.Source = frame.ToBitmap(); } } } // Depth using (var frame = e.OpenDepthImageFrame()) { if (frame != null) { if (_mode == CameraMode.Depth) { camera.Source = frame.ToBitmap(); } } } // Body using (var frame = e.OpenSkeletonFrame()) { if (frame != null) { canvas.Children.Clear(); frame.CopySkeletonDataTo(_bodies); foreach (var body in _bodies) { if (body.TrackingState == SkeletonTrackingState.Tracked) { // COORDINATE MAPPING foreach (Joint joint in body.Joints) { // 3D coordinates in meters SkeletonPoint skeletonPoint = joint.Position; // 2D coordinates in pixels Point point = new Point(); if (_mode == CameraMode.Color) { // Skeleton-to-Color mapping ColorImagePoint colorPoint = _sensor.CoordinateMapper.MapSkeletonPointToColorPoint(skeletonPoint, ColorImageFormat.RgbResolution640x480Fps30); point.X = colorPoint.X; point.Y = colorPoint.Y; } else if (_mode == CameraMode.Depth) // Remember to change the Image and Canvas size to 320x240. { // Skeleton-to-Depth mapping DepthImagePoint depthPoint = _sensor.CoordinateMapper.MapSkeletonPointToDepthPoint(skeletonPoint, DepthImageFormat.Resolution320x240Fps30); point.X = depthPoint.X; point.Y = depthPoint.Y; } // DRAWING... Ellipse ellipse = new Ellipse { Fill = Brushes.LightGray, Width = 20, Height = 20 }; Canvas.SetLeft(ellipse, point.X - ellipse.Width / 2); Canvas.SetTop(ellipse, point.Y - ellipse.Height / 2); canvas.Children.Add(ellipse); } } } } } }
private void KinectAllFramesReady(object sender, AllFramesReadyEventArgs e) { KinectSensor sensor = sender as KinectSensor; foreach (var skeletonCanvas in this.skeletonCanvases) { skeletonCanvas.Skeleton = null; } // Have we already been "shut down" by the user of this viewer, // or has the SkeletonStream been disabled since this event was posted? if ((null == this.KinectSensorManager) || (null == sensor) || (null == sensor.SkeletonStream) || !sensor.SkeletonStream.IsEnabled) { return; } bool haveSkeletonData = false; long frameTimeStamp = -1; using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { if (skeletonFrame != null) { if ((this.skeletonData == null) || (this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } skeletonFrame.CopySkeletonDataTo(this.skeletonData); frameTimeStamp = skeletonFrame.Timestamp; haveSkeletonData = true; } } int trackedIndex = -1; // find the first tracked skeleton and set var trackedSkeleton accordingly for (int i = 0; i < skeletonData.Length; i++) { if (skeletonData[i].TrackingState.Equals(SkeletonTrackingState.Tracked)) { trackedIndex = i; break; } } bool isFullyTracked = false; if (isMeasuring && trackedIndex > -1) { // check to see if the skeleton @ trackedIndex is fully tracked if (fullyTrackedMapping == null && IsFullyTracked(skeletonData[trackedIndex])) { isFullyTracked = true; } SkeletonMeasurer measurer = new SkeletonMeasurer(skeletonData[trackedIndex]); measurer.determineMeasurements(); AddMeasurementsToBuffer(measurer.TestMeasurements); skeletonBuffer.Add(ObjectCopier.Clone <Skeleton>(skeletonData[trackedIndex])); frameTimeStampBuffer.Add(frameTimeStamp); } if (haveSkeletonData) { ColorImageFormat colorFormat = ColorImageFormat.Undefined; int colorWidth = 0; int colorHeight = 0; DepthImageFormat depthFormat = DepthImageFormat.Undefined; int depthWidth = 0; int depthHeight = 0; switch (this.ImageType) { case ImageType.Color: // Retrieve the current color format, from the frame if present, and from the sensor if not. using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame()) { if (null != colorImageFrame) { colorFormat = colorImageFrame.Format; colorWidth = colorImageFrame.Width; colorHeight = colorImageFrame.Height; } else if (null != sensor.ColorStream) { colorFormat = sensor.ColorStream.Format; colorWidth = sensor.ColorStream.FrameWidth; colorHeight = sensor.ColorStream.FrameHeight; } } break; case ImageType.Depth: // Retrieve the current depth format, from the frame if present, and from the sensor if not. using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { if (null != depthImageFrame) { depthFormat = depthImageFrame.Format; depthWidth = depthImageFrame.Width; depthHeight = depthImageFrame.Height; } else if (null != sensor.DepthStream) { depthFormat = sensor.DepthStream.Format; depthWidth = sensor.DepthStream.FrameWidth; depthHeight = sensor.DepthStream.FrameHeight; } } break; } for (int i = 0; i < this.skeletonData.Length && i < this.skeletonCanvases.Count; i++) { var skeleton = this.skeletonData[i]; var skeletonCanvas = this.skeletonCanvases[i]; var jointMapping = this.jointMappings[i]; jointMapping.Clear(); try { // Transform the data into the correct space // For each joint, we determine the exact X/Y coordinates for the target view foreach (Joint joint in skeleton.Joints) { Point mappedPoint = Get2DPosition( sensor, this.ImageType, this.RenderSize, joint.Position, colorFormat, colorWidth, colorHeight, depthFormat, depthWidth, depthHeight); jointMapping[joint.JointType] = new JointMapping { Joint = joint, MappedPoint = mappedPoint }; } } catch (UnauthorizedAccessException) { // Kinect is no longer available. return; } // Look up the center point Point centerPoint = Get2DPosition( sensor, this.ImageType, this.RenderSize, skeleton.Position, colorFormat, colorWidth, colorHeight, depthFormat, depthWidth, depthHeight); // Scale the skeleton thickness // 1.0 is the desired size at 640 width double scale = this.RenderSize.Width / 640; skeletonCanvas.Skeleton = skeleton; skeletonCanvas.JointMappings = jointMapping; skeletonCanvas.Center = centerPoint; skeletonCanvas.ScaleFactor = scale; } if (isFullyTracked) { fullyTrackedMapping = new Dictionary <JointType, JointMapping>(); foreach (JointType type in jointMappings[trackedIndex].Keys) { fullyTrackedMapping[type] = new JointMapping(); fullyTrackedMapping[type].Joint = jointMappings[trackedIndex][type].Joint; fullyTrackedMapping[type].MappedPoint = jointMappings[trackedIndex][type].MappedPoint; } } } }
// Handle Privacy Screen private void KinectSensorOnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { var colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); var depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); //var skeltImageFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); try { using (var skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame()) { if (null != skeletonFrame) { skeletonFrame.CopySkeletonDataTo(this.skeletons); ChoosePrivacySkeleton(); } } { if (colorImageFrame == null) { return; } if (depthImageFrame == null) { return; } // if (skeltImageFrame == null) // { // return; // } // Console.WriteLine(s); depthRange = 300; if (s != null) { double min= 50; double max = 0; depthRange = 300; foreach (Joint joint in s.Joints) { ColorImagePoint point = ks.CoordinateMapper.MapSkeletonPointToColorPoint(joint.Position, ColorImageFormat.RgbResolution640x480Fps30); if (point.X > -1000000 && point.X < 1000000 && point.Y > -1000000 && point.Y < 1000000) { Joint j = joint; double depth = joint.Position.Z; if (depth < min) { min = depth; } if (depth > max) { max = depth; } } } currentDepth = min * 1000 - 800 - 100 - 50; depthRange = (int)(max * 1000 - 800 - 100 - currentDepth); /* Joint jointHead = s.Joints[JointType.Head]; Joint jointCenter = s.Joints[JointType.ShoulderCenter]; ColorImagePoint pointHead = ks.CoordinateMapper.MapSkeletonPointToColorPoint(jointHead.Position, ColorImageFormat.RgbResolution640x480Fps30); ColorImagePoint pointCenter = ks.CoordinateMapper.MapSkeletonPointToColorPoint(jointCenter.Position, ColorImageFormat.RgbResolution640x480Fps30); Console.WriteLine(jointHead.Position.Z * 1000); if (pointHead.X > -1000000 && pointHead.X < 1000000 && pointHead.Y > -1000000 && pointHead.Y < 1000000) { currentDepth = jointHead.Position.Z * 1000 - 700 - 100; depthRange = 200; } else if (pointCenter.X > -1000000 && pointCenter.X < 1000000 && pointCenter.Y > -1000000 && pointCenter.Y < 1000000) { currentDepth = jointCenter.Position.Z * 1000 - 700 - 100; depthRange = 200; }*/ } DepthImagePixel[] depthPixels = new DepthImagePixel[depthImageFrame.PixelDataLength]; depthImageFrame.CopyDepthImagePixelDataTo(depthPixels); int minDepth = depthImageFrame.MinDepth; int maxDepth = depthImageFrame.MaxDepth; var ratio = colorImageFrame.PixelDataLength / depthImageFrame.PixelDataLength; var heightC = colorImageFrame.Height; var heightD = depthImageFrame.Height; var LengthC = colorImageFrame.PixelDataLength; var LengthD = depthPixels.Length; var ratH = colorImageFrame.Height / depthImageFrame.Height; var ratW = colorImageFrame.Width / depthImageFrame.Width; // Make a copy of the color frame for displaying. var haveNewFormat = this.currentColorImageFormat != colorImageFrame.Format; if (haveNewFormat) { this.currentColorImageFormat = colorImageFrame.Format; this.colorImageData = new byte[colorImageFrame.PixelDataLength]; this.colorImageWritableBitmap = new WriteableBitmap( colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null); ColorImage.Source = this.colorImageWritableBitmap; } colorImageFrame.CopyPixelDataTo(this.colorImageData); newColorImageData = new byte[this.colorImageData.Length]; int tempMinDepth = (int)(minDepth + currentDepth); int tempMaxDepth = (int)(minDepth + currentDepth + depthRange); for (int i = 0; i < colorImageFrame.Width; ++i) { int srcX = i / ratW; for (int j = 0; j < colorImageFrame.Height; ++j) { int srcY = j / ratH; int srcPixel = srcX + 2 + ((srcY - 15) * depthImageFrame.Width); int tgtPixel = (i + (j * colorImageFrame.Width)); int l = depthPixels.Length; if (srcPixel >= 0 && srcPixel < l) { // currentDepth = currentDepth + .00001; short depth = depthPixels[(int)srcPixel].Depth; /* if (depth < tempMinDepth) { //changePixel(tgtPixel, 0); int index = tgtPixel * 4; this.colorImageData[index++] = 0; this.colorImageData[index++] = 0; this.colorImageData[index++] = 0; //changePixel(tgtPixel, new byte[]{255, 255, 255}); } //else if (depth > maxDepth) else if (depth > tempMaxDepth) { // changePixel(tgtPixel, 0); int index = tgtPixel * 4; this.colorImageData[index++] = 0; this.colorImageData[index++] = 0; this.colorImageData[index++] = 0; } else { //changePixel(tgtPixel, 0); } if (currentDepth + depthRange >= maxDepth + 300) { currentDepth = minDepth; } */ if (depth > tempMinDepth && depth < tempMaxDepth) { int index = tgtPixel * 4; newColorImageData[index] = this.colorImageData[index++]; newColorImageData[index] = this.colorImageData[index++]; newColorImageData[index] = this.colorImageData[index++]; } } } } /* this.colorImageWritableBitmap.WritePixels( new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height), this.colorImageData, colorImageFrame.Width * Bgr32BytesPerPixel, 0); // Write the pixel data into our bitmap this.foregroundBitmap.WritePixels( new Int32Rect(0, 0, this.foregroundBitmap.PixelWidth, this.foregroundBitmap.PixelHeight), backgroundRemovedFrame.GetRawPixelData(), this.foregroundBitmap.PixelWidth * sizeof(int), 0); */ this.colorImageWritableBitmap.WritePixels( new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height), this.newColorImageData, colorImageFrame.Width * sizeof(int), 0); } } finally { if (colorImageFrame != null) { colorImageFrame.Dispose(); } if (depthImageFrame != null) { depthImageFrame.Dispose(); } // if (skeltImageFrame != null) // { // skeltImageFrame.Dispose(); // } } // Console.WriteLine(s); // ChoosePrivacySkeleton(); }
private void FaceTrackOperation(object sender, AllFramesReadyEventArgs e) { using (var colorImageFrame = e.OpenColorImageFrame()) { if (colorImageFrame == null) { return; } // Make a copy of the color frame for displaying. var haveNewFormat = this.currentColorImageFormat != colorImageFrame.Format; if (haveNewFormat) { this.currentColorImageFormat = colorImageFrame.Format; this.colorImageData = new byte[colorImageFrame.PixelDataLength]; this.colorImageWritableBitmap = new WriteableBitmap( colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null); ColorImage.Source = this.colorImageWritableBitmap; } colorImageFrame.CopyPixelDataTo(this.colorImageData); this.colorImageWritableBitmap.WritePixels( new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height), this.colorImageData, colorImageFrame.Width * Bgr32BytesPerPixel, 0); } using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame()) { if (colorImageFrame == null) return; colorImageFrame.CopyPixelDataTo(colorPixelData); } using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { if (depthImageFrame == null) return; depthImageFrame.CopyPixelDataTo(depthPixelData); } using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { if (skeletonFrame == null) return; skeletonFrame.CopySkeletonDataTo(skeletonData); } var skeleton = skeletonData.FirstOrDefault(s => s.TrackingState == SkeletonTrackingState.Tracked); if (skeleton == null) return; FaceTrackFrame faceFrame = faceTracker.Track(sensorChooser.Kinect.ColorStream.Format, colorPixelData, sensorChooser.Kinect.DepthStream.Format, depthPixelData, skeleton); // If a face is tracked, then we can use it. if (faceFrame.TrackSuccessful) { // Retrieve only the Animation Units coeffs. //CanvasTranslate.X = faceFrame.Translation.X; //CanvasTranslate.Y = faceFrame.Translation.Y; Vector3DF faceRotation = faceFrame.Rotation; var AUCoeff = faceFrame.GetAnimationUnitCoefficients(); var jawLower = AUCoeff[AnimationUnit.JawLower]; var BrowLower = AUCoeff[AnimationUnit.BrowLower]; var BrowUpper = AUCoeff[AnimationUnit.BrowRaiser]; var lcd = AUCoeff[AnimationUnit.LipCornerDepressor]; var lipRaiser = AUCoeff[AnimationUnit.LipRaiser]; var lipStrectch = AUCoeff[AnimationUnit.LipStretcher]; var Pitch = faceRotation.X - midPitch; var Yaw = faceRotation.Y - midYaw; var Roll = faceRotation.Z - midRoll; if (Pitch < midPitch + 10 && Pitch > midPitch - 10) { isHeadDown = 0; state[1] += 3; } if (Roll < midRoll + 10 && Roll > midRoll - 10) { isTilt = 0; state[1]++; } if (Yaw < midYaw + 10 && Yaw > midYaw - 10) { isHeadAway = 0; state[1]++; } if (Pitch > midPitch+10) { if (isHeadDown <= 0) timesNodded++; isHeadDown = 1; state[0]+=3; } if (Pitch < midPitch-10) { if (isHeadDown >= 0) timesNodded++; isHeadDown = -1; state[2]+=3; //timesNodded++; } if (Yaw > midYaw+10 ) { if (isHeadAway <= 0) timesShook++; isHeadAway = 1; state[3]+=3; //timesShook++; } if (Yaw < midYaw-10 ) { if (isHeadAway >= 0) timesShook++; isHeadAway = -1; //timesShook++; state[5]+=3; } if (Roll > midRoll+10) { state[6]+=3; isTilt = 1; //timesShook++; } if (Roll < midRoll-10) { state[8]+=3; isTilt = -1; //timesShook++; } //textBox2.Text = "P: " + ((float)Pitch).ToString() + " Y: " + ((float)Yaw).ToString() + " R: " + ((float)Roll).ToString(); //textBox2.Text = "JL: " + ((float)jawLower).ToString() + " BL: " + ((float)BrowLower).ToString() + " BU: " + ((float)BrowUpper).ToString(); //dataToBeSent3 = "lcd: " + ((float)lcd).ToString() + " LR: " + ((float)lipRaiser).ToString() + " LS: " + ((float)lipStrectch).ToString(); //pitch(nod) - +-15 roll(tilt) - +-20 yaw(shaking offs- -10) +-15 //textBox2.Text += " isTilt " + isTilt + " isHeadAway " + isHeadAway + " isHeadDown " + isHeadDown; } }
private void nui_AllFramesReady(object sender, AllFramesReadyEventArgs e) { //DepthStream과 SkeletonStream에 동시에 추가한 이벤트 핸들러 SkeletonFrame sf = e.OpenSkeletonFrame(); //현재 인식한 골격정보(스캘리톤)을 sf라는 변수에 저장 if (sf == null) { return; } //골격이 인식되지 못하면, 즉 사람이 인식되지 않으면 보여줄 상자, 이를 이은 선을 보여줄 필요가 없으므로 종료 Skeleton[] skeletonData = new Skeleton[sf.SkeletonArrayLength]; //스캘리톤(골격정보)를 저장하라 배열을 인식된 수만큼 크기 지정 sf.CopySkeletonDataTo(skeletonData); //현재 SkeletonFrame안에서 스캘리톤 데이터를 복사 using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { //현재 DepthImageFrame 안에서 데이터 복사 if (depthImageFrame != null) { //데이터가 있다면 foreach (Skeleton sd in skeletonData) { //복사해온 스캘리톤 데이터를 하나씩 차근차근 반복한다 if (sd.TrackingState == SkeletonTrackingState.Tracked) { //인식된 사용자 인덱스 중 추적이 되는 Tracked에 해당하는 골격정보만 사용 int nMax = 20; //키넥트가 인식하는 관절 포인트가 20개 이므로 Joint[] joints = new Joint[nMax]; //만들 조인트만큼(키넥트가 인식하는 것만큼> 20) 조인트 배열 선언 for (int j = 0; j < nMax; j++) { //조인트 생성(초기화 및 설정) joints[j] = sd.Joints[(JointType)j]; } //여기까지 찾아진 골격상태에서 각 골격의 정보를 얻어오기 위함 Point[] points = new Point[nMax]; //뼈대 위치를 저장할 배열 for (int j = 0; j < nMax; j++) { DepthImagePoint depthPoint; depthPoint = depthImageFrame.MapFromSkeletonPoint(joints[j].Position); points[j] = new Point((int)((user_img.Width * depthPoint.X / depthImageFrame.Width) + 340), (int)(user_img.Height * depthPoint.Y / depthImageFrame.Height)); } //여기까지 각 조인트정보로부터 좌표를 얻기 위함 for (int j = 0; j < nMax; j++) { m_rect[j].Visibility = Visibility.Visible; //아까 만들었던 사각형을 보이도록 함 Canvas.SetTop(m_rect[j], points[j].Y - (m_rect[j].Height / 2)); Canvas.SetLeft(m_rect[j], points[j].X - (m_rect[j].Width / 2)); //사각형 배치 } //여기까지 가져온 각 조인트의 정보를 화면에 표시하는 코드 //엉덩이 중앙부터 머리까지 연결되는 선을 긋는 코드 PointCollection pc0 = new PointCollection(4); //이을 관절 개수만큼 크기를 지정하고 위치를 저장할 변수 pc0.Add(points[(int)JointType.HipCenter]); //엉덩이 중간 pc0.Add(points[(int)JointType.Spine]); //등뼈 pc0.Add(points[(int)JointType.ShoulderCenter]); //어깨 중간 pc0.Add(points[(int)JointType.Head]); //머리 m_poly[0].Points = pc0; //관절들 위치를 저장한 변수를 선으로 그리는 함수로 선언한 배열에 넣는다 m_poly[0].Visibility = Visibility.Visible; //넣었으니 활성화시켜 위치를 토대로 그린 선을 보여준다 //왼쪽 손부터 어깨까지 연결되는 선을 긋는 코드 PointCollection pc1 = new PointCollection(5); pc1.Add(points[(int)JointType.ShoulderCenter]); //어깨 중간 pc1.Add(points[(int)JointType.ShoulderLeft]); //왼쪽 어깨 pc1.Add(points[(int)JointType.ElbowLeft]); //왼쪽 팔꿈치 pc1.Add(points[(int)JointType.WristLeft]); //왼쪽 손목 pc1.Add(points[(int)JointType.HandLeft]); //왼손 m_poly[1].Points = pc1; m_poly[1].Visibility = Visibility.Visible; //오른쪽 손부터 어깨까지 연결되는 선을 긋는 코드 PointCollection pc2 = new PointCollection(5); pc2.Add(points[(int)JointType.ShoulderCenter]); //어깨 중간 pc2.Add(points[(int)JointType.ShoulderRight]); //오른쪽 어깨 pc2.Add(points[(int)JointType.ElbowRight]); //오른쪽 팔꿈치 pc2.Add(points[(int)JointType.WristRight]); //오른쪽 손목 pc2.Add(points[(int)JointType.HandRight]); //오른손 m_poly[2].Points = pc2; m_poly[2].Visibility = Visibility.Visible; //왼발부터 엉덩이까지 연결되는 선을 긋는 코드 PointCollection pc3 = new PointCollection(5); pc3.Add(points[(int)JointType.HipCenter]); //엉덩이 중간 pc3.Add(points[(int)JointType.HipLeft]); //엉덩이 왼쪽 pc3.Add(points[(int)JointType.KneeLeft]); //왼쪽 무릎 pc3.Add(points[(int)JointType.AnkleLeft]); //왼쪽 발목 pc3.Add(points[(int)JointType.FootLeft]); //왼쪽 발 m_poly[3].Points = pc3; m_poly[3].Visibility = Visibility.Visible; //오른발부터 엉덩이까지 연결되는 선을 긋는 코드 PointCollection pc4 = new PointCollection(5); pc4.Add(points[(int)JointType.HipCenter]); //엉덩이 중간 pc4.Add(points[(int)JointType.HipRight]); //엉덩이 오른쪽 pc4.Add(points[(int)JointType.KneeRight]); //오른쪽 무릎 pc4.Add(points[(int)JointType.AnkleRight]); //오른쪽 발목 pc4.Add(points[(int)JointType.FootRight]); //오른쪽 발 m_poly[4].Points = pc4; m_poly[4].Visibility = Visibility.Visible; } } } } }
void kinectSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { // Retrieve each single frame and copy the data using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame()) { if (colorImageFrame == null) { return; } colorImageFrame.CopyPixelDataTo(colorPixelData); //int strade = colorImageFrame.Width * 4; //image1.Source = BitmapSource.Create(colorImageFrame.Width, colorImageFrame.Height, 96, 96, // PixelFormats.Bgr32, null, colorPixelData, strade); } using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { if (depthImageFrame == null) { return; } depthImageFrame.CopyPixelDataTo(depthPixelData); } using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { if (skeletonFrame == null) { return; } skeletonFrame.CopySkeletonDataTo(skeletonData); } // Retrieve the first tracked skeleton if any. Otherwise, do nothing. var skeleton = skeletonData.FirstOrDefault(s => s.TrackingState == SkeletonTrackingState.Tracked); if (skeleton == null && !sessionClose) { serialP.WriteLine("s"); serialP.WriteLine("c"); serialP.WriteLine("p"); serialP.WriteLine("g"); if (isActive) { isActive = false; } slejenie = false; activatorRightHand = 0; activatorLeftHand = false; firstMeet = false; sessionClose = true; return; } else if (skeleton != null && !firstMeet) { serialP.WriteLine("i"); playsound(comms[0]); firstMeet = true; sessionClose = false; } if (sessionClose) { return; } // Make the faceTracker processing the data. FaceTrackFrame faceFrame = faceTracker.Track(kinectSensor.ColorStream.Format, colorPixelData, kinectSensor.DepthStream.Format, depthPixelData, skeleton); EnumIndexableCollection <FeaturePoint, PointF> facePoints = faceFrame.GetProjected3DShape(); // points of hands and shoulder - to determine HELLO, etc. Joint shoulderCenter = skeleton.Joints[JointType.ShoulderCenter]; Joint head = skeleton.Joints[JointType.Head]; Joint rightHand = skeleton.Joints[JointType.HandRight]; Joint leftHand = skeleton.Joints[JointType.HandLeft]; // initialize sound for hello //SoundPlayer a = new SoundPlayer("C:\\sal.wav"); // open stream for uart reading //serialP.Open(); // points of lip's corner - with help of this I determine smile double x1 = facePoints[88].X; double y1 = facePoints[88].Y; System.Windows.Point leftLip = new System.Windows.Point(x1, y1); double x2 = facePoints[89].X; double y2 = facePoints[89].Y; System.Windows.Point rightLip = new System.Windows.Point(x2, y2); Vector subtr = System.Windows.Point.Subtract(leftLip, rightLip); // distance between kinect and human distance = skeleton.Position.Z * 100; // distance between two corners of lip double length = Math.Sqrt(subtr.X * subtr.X + subtr.Y * subtr.Y); int check = 100; double angle1 = 0d; double angle2 = 0d; double angle = skeleton.Position.X * 100; #region "Smile deterine" if (distance >= 95 && distance < 110) { check = 22; } else if (distance >= 110 && distance < 120) { check = 19; } else if (distance >= 120 && distance < 130) { check = 18; } else if (distance >= 130 && distance < 140) { check = 17; } else if (distance >= 140 && distance < 150) { check = 16; } else if (distance >= 150 && distance < 160) { check = 14; } else if (distance >= 160 && distance < 170) { check = 13; } else if (distance >= 170 && distance < 180) { check = 12; } else if (distance >= 180 && distance < 190) { check = 11; } #endregion #region "Angle" if (distance >= 90 && distance < 110) { angle1 = -15; angle2 = 15; } else if (distance >= 110 && distance < 150) { angle1 = -20; angle2 = 20; } else if (distance >= 150 && distance < 170) { angle1 = -30; angle2 = 30; } else if (distance >= 170 && distance < 200) { angle1 = -35; angle2 = 35; } else if (distance >= 200) { angle1 = -40; angle2 = 40; } #endregion double condition1 = Math.Abs(leftHand.Position.Z * 100 - shoulderCenter.Position.Z * 100); double condition2 = Math.Abs(rightHand.Position.Z * 100 - shoulderCenter.Position.Z * 100); // If position of two hands higher than shoulder it's activate 'slejenie za ob'ektom' if (condition1 > 45 && condition2 > 45 && leftHand.Position.X < rightHand.Position.X) { if (!slejenie) { isActive = true; FIXED_DISTANCE = distance; slejenie = true; } } // The command to stop 'slejenie za ob'ektom' if (leftHand.Position.X > rightHand.Position.X) { isActive = false; } // Slejenie za ob'ektom if (isActive) { int pinkIs = (int)typeCondition.THIRD; int purpleIs = (int)typeCondition.FORTH; int redIs = (int)typeCondition.FIVTH; int yellowIs = (int)typeCondition.SIXTH; if (distance > FIXED_DISTANCE + 10.0d) { if (angle < angle1) { ellipseSmile.Fill = Brushes.Pink; if (currentAction != pinkIs)//povorot na pravo { currentAction = pinkIs; serialP.WriteLine("r"); } } else if (angle > angle2)//povorot na levo { ellipseSmile.Fill = Brushes.Purple; if (currentAction != purpleIs) { currentAction = purpleIs; serialP.WriteLine("l"); } } else { ellipseSmile.Fill = Brushes.Red; if (currentAction != redIs)// vpered { currentAction = redIs; serialP.WriteLine("f"); } } } else if (distance > 90) { if (angle < angle1) { ellipseSmile.Fill = Brushes.Pink; if (currentAction != pinkIs)//na pravo { currentAction = pinkIs; serialP.WriteLine("r"); } } else if (angle > angle2) { ellipseSmile.Fill = Brushes.Purple; if (currentAction != purpleIs)// na levo { currentAction = purpleIs; serialP.WriteLine("l"); } } else { ellipseSmile.Fill = Brushes.Yellow; if (currentAction != yellowIs)//stop, ili - do nothing { currentAction = yellowIs; serialP.WriteLine("s"); } } } else { ellipseSmile.Fill = Brushes.Yellow; if (currentAction != yellowIs)//stop, ili - do nothing { currentAction = yellowIs; serialP.WriteLine("s"); } } } // esli 'slejenie za ob'ektom' otklu4en else if (!isActive) { int blueIs = (int)typeCondition.FIRST; int blackIs = (int)typeCondition.SECOND; int onkol = (int)typeCondition.SEVENTH; if (leftHand.Position.Y > head.Position.Y && rightHand.Position.Y < shoulderCenter.Position.Y) { ellipseSmile.Fill = Brushes.Blue; if (currentAction != blueIs && !activatorLeftHand)//privet levoi rukoi ---------------------------------------------------------------------------- { currentAction = blueIs; serialP.WriteLine("q"); activatorLeftHand = true; } } else if (rightHand.Position.Y > head.Position.Y && leftHand.Position.Y < shoulderCenter.Position.Y) { ellipseSmile.Fill = Brushes.Blue; if (currentAction != onkol && activatorRightHand != 12)//privet pravoi rukoi ----------------------------------------------------------------------------- { currentAction = onkol; serialP.WriteLine("w"); activatorRightHand = 12; } } else { ellipseSmile.Fill = Brushes.Black; if (currentAction != blackIs)// toktaidy ili do nothing { currentAction = blackIs; serialP.WriteLine("s"); } if (currentAction == blackIs) { if (length >= check && currentFace != (int)faceConditions.FIRST) { serialP.WriteLine("z"); // smile currentFace = (int)faceConditions.FIRST; ellipseSmile.Fill = Brushes.Brown; } else if (length < check && currentFace != (int)faceConditions.SECOND) { serialP.WriteLine("x"); // poker face currentFace = (int)faceConditions.SECOND; ellipseSmile.Fill = Brushes.Gold; } #region "povoroti golovoi" if (angle < angle1) { ellipseSmile.Fill = Brushes.Pink; if (!headToRight)//povorot golovi na pravo { headToRight = true; headToCenter = false; headToLeft = false; serialP.WriteLine("k"); } } else if (angle > angle2)//povorot golovi na levo { if (!headToLeft) { headToLeft = true; headToCenter = false; headToRight = false; serialP.WriteLine("j"); } } else if (angle < angle2 && angle > angle1)//golova v centre { if (!headToCenter) { headToCenter = true; headToRight = false; headToLeft = false; serialP.WriteLine("p"); } } #endregion } else if (!faceFrame.TrackSuccessful && currentFace != (int)faceConditions.NONE) { serialP.WriteLine("c"); // sad face currentFace = (int)faceConditions.NONE; ellipseSmile.Fill = Brushes.Chocolate; } } } label2.Content = distance.ToString(); //label1.Content = (leftHand.Position.Z * 100).ToString(); //label3.Content = (shoulderCenter.Position.Z * 100).ToString(); //serialP.Close(); }
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || kinectSensorChooser1.Kinect == null) { return; } //Map a joint location to a point on the depth map //head /*DepthImagePoint headDepthPoint = * depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position);*/ //left hand DepthImagePoint leftDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); //right hand DepthImagePoint rightDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); //Map a depth point to a point on the color image //head /*ColorImagePoint headColorPoint = * depth.MapToColorImagePoint(headDepthPoint.X, headDepthPoint.Y, * ColorImageFormat.RgbResolution640x480Fps30);*/ //left hand ColorImagePoint leftColorPoint = depth.MapToColorImagePoint(leftDepthPoint.X, leftDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right hand ColorImagePoint rightColorPoint = depth.MapToColorImagePoint(rightDepthPoint.X, rightDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //Set location //CameraPosition(headImage, headColorPoint); CameraPosition(leftEllipse, leftColorPoint); CameraPosition(rightEllipse, rightColorPoint); // do the points bool leftValid = false; bool rightValid = false; leftEllipse = doDepthCalculation(leftEllipse, leftDepthPoint, 1250, true, out leftValid); rightEllipse = doDepthCalculation(rightEllipse, rightDepthPoint, 1250, false, out rightValid); // play / turn off the right notes // right hand if (rightValid) { int rectangleIndex = -1; NoteRectangle thisNoteRectangle = getNoteRectangleAtPoint(rightColorPoint, false, out rectangleIndex); thisNoteRectangle.PlayNote(); thisNoteRectangle.Rectangle.Fill = Brushes.Green; rightHandLabel.Content = thisNoteRectangle.Pitch.ToString(); for (int x = 0; x < rightNotes.Count; x++) { if (x != rectangleIndex) { rightNotes[x].StopNote(); rightNotes[x].Rectangle.Fill = null; } } } else { foreach (NoteRectangle x in rightNotes) { x.StopNote(); } int rectangleIndex = -1; getNoteRectangleAtPoint(rightColorPoint, false, out rectangleIndex).StopNote(); } } }
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || kinectSensorChooser1.Kinect == null) { return; } KinectSensor myKinect = kinectSensorChooser1.Kinect; //save coordinates to file SaveCoordinatesAssistant(first, myKinect, myKinect.DepthStream.Format); //Map a joint location to a point on the depth map //head DepthImagePoint headDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.Head].Position, myKinect.DepthStream.Format); //left knee DepthImagePoint leftKneeDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.KneeLeft].Position, myKinect.DepthStream.Format); //right knee DepthImagePoint rightKneeDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.KneeRight].Position, myKinect.DepthStream.Format); //left ankle DepthImagePoint leftAnkleDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.AnkleLeft].Position, myKinect.DepthStream.Format); //right ankle DepthImagePoint rightAnkleDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.AnkleRight].Position, myKinect.DepthStream.Format); //left hip DepthImagePoint leftHipDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.HipLeft].Position, myKinect.DepthStream.Format); //right hip DepthImagePoint rightHipDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.HipRight].Position, myKinect.DepthStream.Format); //shoulder center (used to check for start, bottom, and end of squat) DepthImagePoint shoulderCenterDepthPoint = myKinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.ShoulderCenter].Position, myKinect.DepthStream.Format); //Map a depth point to a point on the color image //head ColorImagePoint headColorPoint = myKinect.CoordinateMapper.MapDepthPointToColorPoint(myKinect.DepthStream.Format, headDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); //left knee ColorImagePoint leftKneeColorPoint = myKinect.CoordinateMapper.MapDepthPointToColorPoint(myKinect.DepthStream.Format, leftKneeDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); //right knee ColorImagePoint rightKneeColorPoint = myKinect.CoordinateMapper.MapDepthPointToColorPoint(myKinect.DepthStream.Format, rightKneeDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); //left ankle ColorImagePoint leftAnkleColorPoint = myKinect.CoordinateMapper.MapDepthPointToColorPoint(myKinect.DepthStream.Format, leftAnkleDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); //right ankle ColorImagePoint rightAnkleColorPoint = myKinect.CoordinateMapper.MapDepthPointToColorPoint(myKinect.DepthStream.Format, rightAnkleDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); //shoulder center ColorImagePoint shoulderCenterColorPoint = myKinect.CoordinateMapper.MapDepthPointToColorPoint(myKinect.DepthStream.Format, shoulderCenterDepthPoint, ColorImageFormat.RgbResolution640x480Fps30); //Set location CameraPosition(headImage, headColorPoint); CameraPosition(leftKneeEllipse, leftKneeColorPoint); CameraPosition(rightKneeEllipse, rightKneeColorPoint); CameraPosition(leftAnkleEllipse, leftAnkleColorPoint); CameraPosition(rightAnkleEllipse, rightAnkleColorPoint); CameraPosition(shoulderCenterEllipse, shoulderCenterColorPoint); //GetPositionInSquat(shoulderCenterDepthPoint.Y, depth.Timestamp); //inSquatDisplay.Text = inSquat.ToString(); double kasr = CalculateKASR(rightAnkleDepthPoint, leftAnkleDepthPoint, rightKneeDepthPoint, leftKneeDepthPoint, rightHipDepthPoint, leftHipDepthPoint); double leftKneeAngle = AngleBetweenJoints(first.Joints[JointType.HipLeft], first.Joints[JointType.KneeLeft], first.Joints[JointType.AnkleLeft]); double rightKneeAngle = AngleBetweenJoints(first.Joints[JointType.HipRight], first.Joints[JointType.KneeRight], first.Joints[JointType.AnkleRight]); //Console.WriteLine("angles: (" + leftKneeAngle + "," + rightKneeAngle + ")"); //add shoulder position to list of previous prevShoulderCenters.Add(shoulderCenterDepthPoint.Y); } }
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; try { colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { return; } // Check for image format changes. The FaceTracker doesn't // deal with that so we need to reset. if (this.depthImageFormat != depthImageFrame.Format) { this.ResetFaceTracking(); this.depthImage = null; this.depthImageFormat = depthImageFrame.Format; } if (this.colorImageFormat != colorImageFrame.Format) { this.ResetFaceTracking(); this.colorImage = null; this.colorImageFormat = colorImageFrame.Format; } // Create any buffers to store copies of the data we work with if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } // Get the skeleton information if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } colorImageFrame.CopyPixelDataTo(this.colorImage); depthImageFrame.CopyPixelDataTo(this.depthImage); skeletonFrame.CopySkeletonDataTo(this.skeletonData); // Update the list of trackers and the trackers with the current frame information foreach (Skeleton skeleton in this.skeletonData) { if (skeleton.TrackingState == SkeletonTrackingState.Tracked || skeleton.TrackingState == SkeletonTrackingState.PositionOnly) { // We want keep a record of any skeleton, tracked or untracked. if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId)) { this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker()); } // Give each tracker the upated frame. SkeletonFaceTracker skeletonFaceTracker; if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker)) { skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton); skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber; } } } this.RemoveOldTrackers(skeletonFrame.FrameNumber); this.InvalidateVisual(); } finally { if (colorImageFrame != null) { colorImageFrame.Dispose(); } if (depthImageFrame != null) { depthImageFrame.Dispose(); } if (skeletonFrame != null) { skeletonFrame.Dispose(); } } }
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || mKinect == null) { return; } DepthImagePoint headDepthPoint = mKinect.Kinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.Head].Position, DepthImageFormat.Resolution640x480Fps30); DepthImagePoint leftHandDepthPoint = mKinect.Kinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandLeft].Position, DepthImageFormat.Resolution640x480Fps30); DepthImagePoint rightHandDepthPoint = mKinect.Kinect.CoordinateMapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30); ColorImagePoint headColorPoint = mKinect.Kinect.CoordinateMapper.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, headDepthPoint, ColorImageFormat.RgbResolution1280x960Fps12); ColorImagePoint leftHandColorPoint = mKinect.Kinect.CoordinateMapper.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, leftHandDepthPoint, ColorImageFormat.RgbResolution1280x960Fps12); ColorImagePoint rightHandColorPoint = mKinect.Kinect.CoordinateMapper.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, rightHandDepthPoint, ColorImageFormat.RgbResolution1280x960Fps12); CameraPosition(ellipseHead, headColorPoint, "HEAD"); CameraPosition(ellipseHandLeft, leftHandColorPoint, "LEFT HAND"); CameraPosition(ellipseHandRight, rightHandColorPoint, "RIGHT HAND"); if (cHandRight == 10) //MANO DERECHA PASO 10 VECES { dHandOnImage = DateTime.Now; Log("----------------------------------------------------------------------------------------------------"); Log("headOnImage - MANO DERECHA"); CameraPosition(ellipseHead, headColorPoint, "HEAD - cHandRight"); CameraPosition(ellipseHandRight, rightHandColorPoint, "RIGHT HAND - cHandRight"); } if (handOnImage(ellipseHandRight, ellipseHead)) { Log("----------------------------------------------------------------------------------------------------"); Log("handOnHead - BIEN - ++ "); cHandRightOnHead++; labelResult.Content = "BIEN ++: " + cHandRightOnHead; labelResult.Visibility = Visibility.Visible; } if (cHandRightOnHead == 10) { Log("BIEN"); labelResult2.Content = "BIEN"; labelResult2.Visibility = Visibility.Visible; imageHeadInBody.Visibility = Visibility.Visible; imageHead.Visibility = Visibility.Hidden; } Log("cHandRightOnHead" + cHandRightOnHead.ToString()); if (cHandLeft == 10) //MANO DERECHA PASO 10 VECES { dHandOnImage = DateTime.Now; Log("----------------------------------------------------------------------------------------------------"); Log("headOnImage - MANO DERECHA"); CameraPosition(ellipseHead, headColorPoint, "HEAD - cHandRight"); CameraPosition(ellipseHandLeft, leftHandColorPoint, "RIGHT HAND - cHandRight"); } if (handOnImage(ellipseHandLeft, ellipseHead)) { Log("----------------------------------------------------------------------------------------------------"); Log("handOnHead - BIEN - ++ "); cHandLeftOnHead++; labelResult.Content = "BIEN ++: " + cHandLeftOnHead; labelResult.Visibility = Visibility.Visible; } if (cHandLeftOnHead == 10) { Log("BIEN"); labelResult2.Content = "BIEN"; labelResult2.Visibility = Visibility.Visible; imageHeadInBody.Visibility = Visibility.Visible; imageHead.Visibility = Visibility.Hidden; } } }
//main game loop and logic; 30fps instead of time ticker private void sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { depthImagePixels = new DepthImagePixel[sensor.DepthStream.FramePixelDataLength]; //on MotionCanvas paints the input from the regular rgb camera using (var frame = e.OpenColorImageFrame()) { if (frame == null) { return; } var bitmap = CreateBitmap(frame); MotionCanvas.Background = new ImageBrush(bitmap); } //on ImageCanvas changes the layer color / everything else remains black using (var frame = e.OpenDepthImageFrame()) { depthImagePixels = new DepthImagePixel[sensor.DepthStream.FramePixelDataLength]; try { frame.CopyDepthImagePixelDataTo(depthImagePixels); } catch (Exception ex) { Console.WriteLine(ex); } var colorPixels = new byte[4 * sensor.DepthStream.FramePixelDataLength]; // 4byte/pixel for (int i = 0; i < colorPixels.Length; i += 4) { if (depthImagePixels[i / 4].PlayerIndex != 0) { if (r > 10 || g > 10 || b > 10)//slider for custom rgb color, by default green { colorPixels[i + 2] = (byte)r; colorPixels[i + 1] = (byte)g; colorPixels[i + 0] = (byte)b; } else { colorPixels[i + spectre] = 255; } } } ImageCanvas.Background = new ImageBrush(colorPixels.ToBitmapSource(640, 480)); //on MotionCanvas draws the polylines to leave color traces if (isDrawing == true) { trail1.Points.Add(new Point { X = leftHandPoint.X - 70, Y = leftHandPoint.Y - 20 }); trail2.Points.Add(new Point { X = rightHandPoint.X - 150, Y = rightHandPoint.Y - 20 }); trail3.Points.Add(new Point { X = leftFootPoint.X - 70, Y = leftFootPoint.Y - 20 }); trail4.Points.Add(new Point { X = rightFootPoint.X - 150, Y = rightFootPoint.Y - 20 }); } else if (isSet == false && isDrawing == false) { trail1 = new Polyline(); trail1.Stroke = System.Windows.Media.Brushes.Red; trail1.StrokeThickness = 5; trail2 = new Polyline(); trail2.Stroke = System.Windows.Media.Brushes.Blue; trail2.StrokeThickness = 5; trail3 = new Polyline(); trail3.Stroke = System.Windows.Media.Brushes.Green; trail3.StrokeThickness = 5; trail4 = new Polyline(); trail4.Stroke = System.Windows.Media.Brushes.HotPink; trail4.StrokeThickness = 5; isSet = true; MotionCanvas.Children.Clear(); MotionCanvas.Children.Add(trail1); MotionCanvas.Children.Add(trail2); MotionCanvas.Children.Add(trail3); MotionCanvas.Children.Add(trail4); } } //skeleton tracking using (var frame = e.OpenSkeletonFrame()) { if (frame == null) { return; } var skeletons = new Skeleton[frame.SkeletonArrayLength]; frame.CopySkeletonDataTo(skeletons); var skeleton = skeletons.FirstOrDefault(s => s.TrackingState == SkeletonTrackingState.Tracked); if (skeleton == null) { return; } var rightHand = skeleton.Joints[JointType.HandRight].Position; var leftHand = skeleton.Joints[JointType.HandLeft].Position; var rightLeg = skeleton.Joints[JointType.FootRight].Position; var leftLeg = skeleton.Joints[JointType.FootLeft].Position; var mapper = new CoordinateMapper(sensor); var colorPoint = mapper.MapSkeletonPointToColorPoint(rightHand, ColorImageFormat.RawBayerResolution640x480Fps30); var colorPoint2 = mapper.MapSkeletonPointToColorPoint(rightLeg, ColorImageFormat.RawBayerResolution640x480Fps30); var colorPoint3 = mapper.MapSkeletonPointToColorPoint(leftLeg, ColorImageFormat.RawBayerResolution640x480Fps30); var colorPoint4 = mapper.MapSkeletonPointToColorPoint(leftHand, ColorImageFormat.RawBayerResolution640x480Fps30); //hand/feet positions leftHandPoint.X = colorPoint4.X; leftHandPoint.Y = colorPoint4.Y; rightHandPoint.X = colorPoint.X; rightHandPoint.Y = colorPoint.Y; rightFootPoint.X = colorPoint2.X; rightFootPoint.Y = colorPoint2.Y; leftFootPoint.X = colorPoint3.X; leftFootPoint.Y = colorPoint3.Y; var pointList = new List <ColorImagePoint>(); pointList.Add(colorPoint); pointList.Add(colorPoint2); pointList.Add(colorPoint3); pointList.Add(colorPoint4); BulletBounce(pointList); var circle = CreateCircle(colorPoint); LevelStart(colorPoint, circle); ProjectileMove(); } }
/// <summary> /// Called whenever the frames are ready. /// Processes the incoming frames using the RGBTracker. /// When processed, calls frameReady for post-frame work & visualisation /// </summary> /// <param name="sender"></param> /// <param name="frames"></param> private void handleNewFrame(object sender, AllFramesReadyEventArgs frames) { //Console.WriteLine("Frame!"); bool run; byte[] thisColorFrame; lock (this) { this.frameCounter++; run = ((frameCounter % gapBetweenFrames) == 0 & (tracking = true)); } // Process the frame? if (run == true) { //Console.WriteLine("Running!!!!!"); using (ColorImageFrame CIF = frames.OpenColorImageFrame()) using (DepthImageFrame DIF = frames.OpenDepthImageFrame()) using (SkeletonFrame SF = frames.OpenSkeletonFrame()) { // Get the frames if (CIF != null & DIF != null) { //Console.WriteLine("Get frames"); thisColorFrame = new byte[640 * 480 * 4]; CIF.CopyPixelDataTo(thisColorFrame); CIF.CopyPixelDataTo(colorFrame); DIF.CopyPixelDataTo(depthFrame); if (SF != null) { skeletonFrame = new Skeleton[SF.SkeletonArrayLength]; SF.CopySkeletonDataTo(skeletonFrame); int skeletons = 0; foreach (var skeleton in skeletonFrame) { if (skeleton.TrackingState == SkeletonTrackingState.Tracked) { skeletons++; } } activeSkeletons = skeletons; } } else { // Frames automatically disposed via using statement return; } // Now the frame data has been captured, process it! // Need to use local copies to maintain volatility of actual variables int tempX = 0; int tempY = 0; double tempAngle = 0; // Get the values byte[] colorFrame2 = (byte[])thisColorFrame.Clone(); //new Thread(() => tracker.ProcessFrame(colorFrame2, out tempX, out tempY, out tempAngle); //).Start(); //Console.WriteLine("Valz: " + tempX + ", " + tempY + ", " + tempAngle); // Perform motion smoothing in a function rather than having it all here UpdateCoordinates(tempX, tempY, tempAngle); lock (this) { this.colorFrame = thisColorFrame; } // Post-frame work frameReady(); } } else { //Console.WriteLine("Ignore frame - frames = " + this.frameCounter); // Ignore the frame & dispose using (ColorImageFrame CIF = frames.OpenColorImageFrame()) using (DepthImageFrame DIF = frames.OpenDepthImageFrame()) using (SkeletonFrame SF = frames.OpenSkeletonFrame()) { try { // But, get the latest colour frame for display, first! if (frames.OpenColorImageFrame() != null) { frames.OpenColorImageFrame().CopyPixelDataTo(colorFrame); } } catch (NullReferenceException Frame_Is_Empty) { // Try to ignore it? } } updateVisualisation(); } }
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e) { using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null) { return; } //Map a joint location to a point on the depth map //head DepthImagePoint headDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); //left shoulder DepthImagePoint leftSDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.ShoulderLeft].Position); //right shoulder DepthImagePoint rightSDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.ShoulderRight].Position); //right elbow DepthImagePoint rightEDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.ElbowRight].Position); //left elbow DepthImagePoint leftEDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.ElbowLeft].Position); //right hand DepthImagePoint rightHdDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); //left hand DepthImagePoint leftHdDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); //spine DepthImagePoint spineDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Spine].Position); //right hip DepthImagePoint rightHpDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HipRight].Position); //left hip DepthImagePoint leftHpDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HipLeft].Position); //hip centre DepthImagePoint centreHpDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.HipCenter].Position); //right knee DepthImagePoint rightKDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.KneeRight].Position); //left knee DepthImagePoint leftKDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.KneeLeft].Position); //right ankle DepthImagePoint rightADepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.AnkleRight].Position); //left ankle DepthImagePoint leftADepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.AnkleLeft].Position); //Map a depth point to a point on the color image //head ColorImagePoint headColorPoint = depth.MapToColorImagePoint(headDepthPoint.X, headDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left shoulder ColorImagePoint leftShoulderColorPoint = depth.MapToColorImagePoint(leftSDepthPoint.X, leftSDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right shoulder ColorImagePoint rightShoulderColorPoint = depth.MapToColorImagePoint(rightSDepthPoint.X, rightSDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right elbow ColorImagePoint rightEColorPoint = depth.MapToColorImagePoint(rightEDepthPoint.X, rightEDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left eblow ColorImagePoint leftEColorPoint = depth.MapToColorImagePoint(leftEDepthPoint.X, leftEDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right hand ColorImagePoint rightHdColorPoint = depth.MapToColorImagePoint(rightHdDepthPoint.X, rightHdDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left hand ColorImagePoint leftHdColorPoint = depth.MapToColorImagePoint(leftHdDepthPoint.X, leftHdDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //spine ColorImagePoint spineColorPoint = depth.MapToColorImagePoint(spineDepthPoint.X, spineDepthPoint.Y, ColorImageFormat.RawBayerResolution640x480Fps30); //right hip ColorImagePoint rightHpColorPoint = depth.MapToColorImagePoint(rightHpDepthPoint.X, rightHpDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left hip ColorImagePoint leftHpColorPoint = depth.MapToColorImagePoint(leftHpDepthPoint.X, leftHpDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //centre hip ColorImagePoint centreHpColorPoint = depth.MapToColorImagePoint(centreHpDepthPoint.X, centreHpDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right knee ColorImagePoint rightKColorPoint = depth.MapToColorImagePoint(rightKDepthPoint.X, rightKDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left knee ColorImagePoint leftKColorPoint = depth.MapToColorImagePoint(leftKDepthPoint.X, leftKDepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //right ankle ColorImagePoint rightAColorPoint = depth.MapToColorImagePoint(rightADepthPoint.X, rightADepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //left ankle ColorImagePoint leftAColorPoint = depth.MapToColorImagePoint(leftADepthPoint.X, leftADepthPoint.Y, ColorImageFormat.RgbResolution640x480Fps30); //Set Points jointPointDict.AddOrUpdate("Head", headColorPoint, (key, oldValue) => headColorPoint); jointPointDict.AddOrUpdate("ShoulderLeft", leftShoulderColorPoint, (key, oldValue) => leftShoulderColorPoint); jointPointDict.AddOrUpdate("ShoulderRight", rightShoulderColorPoint, (key, oldValue) => rightShoulderColorPoint); jointPointDict.AddOrUpdate("ElbowRight", rightEColorPoint, (key, oldValue) => rightEColorPoint); jointPointDict.AddOrUpdate("ElbowLeft", leftEColorPoint, (key, oldValue) => leftEColorPoint); jointPointDict.AddOrUpdate("HandRight", rightHdColorPoint, (key, oldValue) => rightHdColorPoint); jointPointDict.AddOrUpdate("HandLeft", leftHdColorPoint, (key, oldValue) => leftHdColorPoint); jointPointDict.AddOrUpdate("Spine", spineColorPoint, (key, oldValue) => spineColorPoint); jointPointDict.AddOrUpdate("HipRight", rightHpColorPoint, (key, oldValue) => rightHpColorPoint); jointPointDict.AddOrUpdate("HipLeft", leftHpColorPoint, (key, oldValue) => leftHpColorPoint); jointPointDict.AddOrUpdate("HipCentre", centreHpColorPoint, (key, oldValue) => centreHpColorPoint); jointPointDict.AddOrUpdate("KneeRight", rightKColorPoint, (key, oldValue) => rightKColorPoint); jointPointDict.AddOrUpdate("KneeLeft", leftKColorPoint, (key, oldValue) => leftKColorPoint); jointPointDict.AddOrUpdate("AnkleRight", rightAColorPoint, (key, oldValue) => rightAColorPoint); jointPointDict.AddOrUpdate("AnkleLeft", leftAColorPoint, (key, oldValue) => leftAColorPoint); } }
void nui_AllFramesReady(object sender, AllFramesReadyEventArgs e) { ColorImageFrame cf = e.OpenColorImageFrame(); DepthImageFrame df = e.OpenDepthImageFrame(); SkeletonFrame sf = e.OpenSkeletonFrame(); // Skeleton[] skeletonData = new Skeleton[sf.SkeletonArrayLength]; if (cf == null || df == null || sf == null) { return; } byte[] ImageBits = new byte[cf.PixelDataLength]; cf.CopyPixelDataTo(ImageBits); BitmapSource src = null; src = BitmapSource.Create(cf.Width, cf.Height, 96, 96, PixelFormats.Bgr32, null, ImageBits, cf.Width * cf.BytesPerPixel); image2.Source = src; // Check for image format changes. The FaceTracker doesn't // deal with that so we need to reset. if (this.depthImageFormat != df.Format) { this.ResetFaceTracking(); this.depthImage = null; this.depthImageFormat = df.Format; } if (this.colorImageFormat != cf.Format) { this.ResetFaceTracking(); this.colorImage = null; this.colorImageFormat = cf.Format; } // Create any buffers to store copies of the data we work with if (this.depthImage == null) { this.depthImage = new short[df.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[cf.PixelDataLength]; } // Get the skeleton information if (this.skeletonData == null || this.skeletonData.Length != sf.SkeletonArrayLength) { this.skeletonData = new Skeleton[sf.SkeletonArrayLength]; } cf.CopyPixelDataTo(this.colorImage); df.CopyPixelDataTo(this.depthImage); sf.CopySkeletonDataTo(this.skeletonData); foreach (Skeleton skeleton in this.skeletonData) { if (skeleton.TrackingState == SkeletonTrackingState.Tracked || skeleton.TrackingState == SkeletonTrackingState.PositionOnly) { // We want keep a record of any skeleton, tracked or untracked. if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId)) { this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker()); } // Give each tracker the upated frame. SkeletonFaceTracker skeletonFaceTracker; if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker)) { skeletonFaceTracker.OnFrameReady(nui2, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton); skeletonFaceTracker.LastTrackedFrame = sf.FrameNumber; } } } using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame()) { if (depthImageFrame != null) { foreach (Skeleton sd in skeletonData) { if (sd.TrackingState == SkeletonTrackingState.Tracked) { Joint joint = sd.Joints[JointType.Head]; DepthImagePoint depthPoint; // CoordinateMapper coordinateMapper = new CoordinateMapper(nui); // depthPoint = coordinateMapper.MapSkeletonPointToDepthPoint(joint.Position, DepthImageFormat.Resolution320x240Fps30); depthPoint = depthImageFrame.MapFromSkeletonPoint(joint.Position); System.Windows.Point point = new System.Windows.Point((int)(image2.ActualWidth * depthPoint.X / depthImageFrame.Width), (int)(image2.ActualHeight * depthPoint.Y / depthImageFrame.Height)); Canvas.SetLeft(ellipse1, (point.X) - ellipse1.Width); Canvas.SetTop(ellipse1, (point.Y) - ellipse1.Height); App thisApp = App.Current as App; Canvas.SetLeft(rect2, thisApp.m_dbX - rect2.Width); Canvas.SetTop(rect2, thisApp.m_dbY - rect2.Height); double GapX, GapY; GapX = point.X - (thisApp.m_dbX - 2); GapY = point.Y - (thisApp.m_dbY - 2); int siteX = 999, siteY = 999; if (GapX < 30 && GapX > -30) { siteX = 1; } else if (GapX >= 30) { siteX = 0; } else if (GapY <= -30) { siteX = 2; } if (GapY >= -40) { siteY = 0; } else if (GapY < -40 && GapY > -60) { siteY = 1; } else if (GapY <= -60) { siteY = 2; } int site; site = siteX + (siteY * 3); if (site == 0) { text2.Text = "좌상"; } else if (site == 1) { text2.Text = "상"; } else if (site == 2) { text2.Text = "우상"; } else if (site == 3) { text2.Text = "좌"; } else if (site == 4) { text2.Text = "정"; } else if (site == 5) { text2.Text = "우"; } else if (site == 6) { text2.Text = "좌하"; } else if (site == 7) { text2.Text = "하"; } else if (site == 8) { text2.Text = "우하"; } thisApp.nowsite = site; /* * * rect4.X = facePoints[i].X - 2; * rect4.Y = facePoints[i].Y - 2; * rect4.Width = 4; * rect4.Height = 4; */ } } } } }
void sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { if (testnumber == 1) { getSelectedFiles(); startDrag(); testnumber = 0; } drag(); if (closing) { return; } //Get a skeleton Skeleton first = GetFirstSkeleton(e); if (first == null) { return; } //set scaled position //ScalePosition(headImage, first.Joints[JointType.Head]); //ScalePosition(leftEllipse, first.Joints[JointType.HandLeft]); ScalePosition(Cursor, first.Joints[JointType.HandRight]); using (DepthImageFrame depth = e.OpenDepthImageFrame()) { if (depth == null || kinectSensorChooser1.Kinect == null) { return; } float[] skeletonValues = new float[9]; DepthImagePoint leftSkeletalPoint2 = depth.MapFromSkeletonPoint(first.Joints[JointType.HandLeft].Position); DepthImagePoint rightSkeletalPoint2 = depth.MapFromSkeletonPoint(first.Joints[JointType.HandRight].Position); DepthImagePoint headDepthPoint = depth.MapFromSkeletonPoint(first.Joints[JointType.Head].Position); skeletonValues[0] = leftSkeletalPoint2.X; skeletonValues[1] = leftSkeletalPoint2.Y; skeletonValues[2] = leftSkeletalPoint2.Depth; skeletonValues[3] = rightSkeletalPoint2.X; skeletonValues[4] = rightSkeletalPoint2.Y; skeletonValues[5] = rightSkeletalPoint2.Depth; skeletonValues[6] = headDepthPoint.X; skeletonValues[7] = headDepthPoint.Y; skeletonValues[8] = headDepthPoint.Depth; storedSkeletonValues.Add(skeletonValues); System.Windows.Forms.Cursor.Position = new System.Drawing.Point((int)RHPos[0], (int)RHPos[1]); if (!actionWait) { if (!selectActivated) { CheckSwipe(e); } CheckStatic(e); } if (lassoFilesDragging) { //Console.WriteLine("lassofiles dragging"); //myimg_MouseMove(); CursorInCommitBoxZone(); CursorInTrashZone(); } if (selectActivated) { mouseLeftDown(); } } GetCameraPoint(first, e); /*if (selectActivated) * { * FollowPointer(); * } * */ FollowPointer(); }
/// <summary> /// Event handler for Kinect sensor's DepthFrameReady event /// </summary> /// <param name="sender">object sending the event</param> /// <param name="e">event arguments</param> private void SensorAllFramesReady(object sender, AllFramesReadyEventArgs e) { // in the middle of shutting down, so nothing to do if (null == this.sensor) { return; } bool depthReceived = false; bool colorReceived = false; using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (null != depthFrame) { // Copy the pixel data from the image to a temporary array depthFrame.CopyPixelDataTo(this.depthPixels); depthReceived = true; } } using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (null != colorFrame) { // Copy the pixel data from the image to a temporary array colorFrame.CopyPixelDataTo(this.colorPixels); colorReceived = true; } } // do our processing outside of the using block // so that we return resources to the kinect as soon as possible if (true == depthReceived) { this.sensor.MapDepthFrameToColorFrame( DepthFormat, this.depthPixels, ColorFormat, this.colorCoordinates); Array.Clear(this.greenScreenPixelData, 0, this.greenScreenPixelData.Length); // loop over each row and column of the depth for (int y = 0; y < this.depthHeight; ++y) { for (int x = 0; x < this.depthWidth; ++x) { // calculate index into depth array int depthIndex = x + (y * this.depthWidth); short depthPixel = this.depthPixels[depthIndex]; int player = depthPixel & DepthImageFrame.PlayerIndexBitmask; // if we're tracking a player for the current pixel, do green screen if (player > 0) { // retrieve the depth to color mapping for the current depth pixel ColorImagePoint colorImagePoint = this.colorCoordinates[depthIndex]; // scale color coordinates to depth resolution int colorInDepthX = colorImagePoint.X / this.colorToDepthDivisor; int colorInDepthY = colorImagePoint.Y / this.colorToDepthDivisor; // make sure the depth pixel maps to a valid point in color space // check y > 0 and y < depthHeight to make sure we don't write outside of the array // check x > 0 instead of >= 0 since to fill gaps we set opaque current pixel plus the one to the left // because of how the sensor works it is more correct to do it this way than to set to the right if (colorInDepthX > 0 && colorInDepthX < this.depthWidth && colorInDepthY >= 0 && colorInDepthY < this.depthHeight) { // calculate index into the green screen pixel array int greenScreenIndex = colorInDepthX + (colorInDepthY * this.depthWidth); // set opaque this.greenScreenPixelData[greenScreenIndex] = opaquePixelValue; // compensate for depth/color not corresponding exactly by setting the pixel // to the left to opaque as well this.greenScreenPixelData[greenScreenIndex - 1] = opaquePixelValue; } } } } } // do our processing outside of the using block // so that we return resources to the kinect as soon as possible if (true == colorReceived) { // Write the pixel data into our bitmap this.colorBitmap.WritePixels( new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight), this.colorPixels, this.colorBitmap.PixelWidth * sizeof(int), 0); if (this.playerOpacityMaskImage == null) { this.playerOpacityMaskImage = new WriteableBitmap( this.depthWidth, this.depthHeight, 96, 96, PixelFormats.Bgra32, null); MaskedColor.OpacityMask = new ImageBrush { ImageSource = this.playerOpacityMaskImage }; } this.playerOpacityMaskImage.WritePixels( new Int32Rect(0, 0, this.depthWidth, this.depthHeight), this.greenScreenPixelData, this.depthWidth * ((this.playerOpacityMaskImage.Format.BitsPerPixel + 7) / 8), 0); } }
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; try { colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { return; } // Check for image format changes. The FaceTracker doesn't // deal with that so we need to reset. if (this.depthImageFormat != depthImageFrame.Format) { this.resetFaceTracking(); this.depthImage = null; this.depthImageFormat = depthImageFrame.Format; } if (this.colorImageFormat != colorImageFrame.Format) { this.resetFaceTracking(); this.colorImage = null; this.colorImageFormat = colorImageFrame.Format; } // Create any buffers to store copies of the data we work with if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } // Get the skeleton information if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } colorImageFrame.CopyPixelDataTo(this.colorImage); depthImageFrame.CopyPixelDataTo(this.depthImage); skeletonFrame.CopySkeletonDataTo(this.skeletonData); // Update the list of trackers and the trackers with the current frame information foreach (Skeleton skeleton in this.skeletonData) { if (skeleton.TrackingState == SkeletonTrackingState.Tracked || skeleton.TrackingState == SkeletonTrackingState.PositionOnly) { try { this.trackedSkeleton.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton); } catch (NullReferenceException) { //se si perder il tracking del viso si evita un crash } this.trackedSkeleton.LastTrackedFrame = skeletonFrame.FrameNumber; } } this.InvalidateVisual(); } finally { if (colorImageFrame != null) { colorImageFrame.Dispose(); } if (depthImageFrame != null) { depthImageFrame.Dispose(); } if (skeletonFrame != null) { skeletonFrame.Dispose(); } } }
private void AllFramesReadyEventHandler(object sender, AllFramesReadyEventArgs args) { using (ColorImageFrame ciFrame = args.OpenColorImageFrame()) { if (null != ciFrame) { ciFrame.CopyPixelDataTo(this.ColorPixels); ColorBitmap.WritePixels(new Int32Rect(0, 0, ColorWidth, ColorHeight), ColorPixels, ColorWidth * sizeof(int), 0); } } using (DepthImageFrame diFrame = args.OpenDepthImageFrame()) { if (null != diFrame) { diFrame.CopyPixelDataTo(this.DepthDatas); } else { return; } } // Clear //Array.Clear(PlayerPixels, 0, PlayerPixels.Length); //System.Threading.Tasks.Parallel.For(0, PlayerPixels.Length, index => // { // PlayerPixels[index] = 200; // }); Array.Clear(CIP, 0, CIP.Length); gSensor.MapDepthFrameToColorFrame(DIF, DepthDatas, CIF, CIP); byte[] pixels = new byte[gSensor.DepthStream.FramePixelDataLength * sizeof(int)]; // Fill the Player Image for (int hIndex = 0; hIndex < DepthHeight; ++hIndex) { for (int wIndex = 0; wIndex < DepthWidth; ++wIndex) { int index = wIndex + hIndex * DepthWidth; //int player = DepthDatas[index] & DepthImageFrame.PlayerIndexBitmask; if (0 < (DepthDatas[index] & DepthImageFrame.PlayerIndexBitmask)) // Just for Player { ColorImagePoint cip = CIP[index]; // scale color coordinates to depth resolution int colorInDepthX = (int)(cip.X / this.Divisor); int colorInDepthY = (int)(cip.Y / this.Divisor); if (colorInDepthX > 0 && colorInDepthX < this.DepthWidth && colorInDepthY >= 0 && colorInDepthY < this.DepthHeight) { // calculate index into the green screen pixel array int playerIndex = (colorInDepthX + (colorInDepthY * this.DepthWidth)) << 2; int colorIndex = (cip.X + cip.Y * ColorWidth) << 2; pixels[playerIndex] = ColorPixels[colorIndex]; //BitConverter.ToInt32(ColorPixels, colorIndex); pixels[playerIndex + 1] = ColorPixels[colorIndex + 1]; pixels[playerIndex + 2] = ColorPixels[colorIndex + 2]; pixels[playerIndex + 3] = ColorPixels[colorIndex + 3]; --playerIndex; --colorIndex; pixels[playerIndex] = ColorPixels[colorIndex]; //BitConverter.ToInt32(ColorPixels, colorIndex); pixels[playerIndex + 1] = ColorPixels[colorIndex + 1]; pixels[playerIndex + 2] = ColorPixels[colorIndex + 2]; pixels[playerIndex + 3] = ColorPixels[colorIndex + 3]; } HadPlayer = true; } //else //{ // HadPlayer = false; //} } } lock (gLock) { // Enqueue //PixelsQueue.Enqueue(pixels); //Average.ResetQueue(PixelsQueue, 3); PixelsLinkedList.AddLast(pixels); Average.ResetLinkedList(PixelsLinkedList, 3); } // Smoothen if (null == smooth && HadPlayer) { Color bg = new Color(); bg.B = bg.G = bg.R = 0; // Gaussian //smooth = new GaussianFilter(DepthWidth, DepthHeight, PixelFormats.Bgr32, bg); // Bilateral smooth = new BilateralFilter(DepthWidth, DepthHeight, PixelFormats.Bgr32); // Median smooth2 = new GenericMedian(DepthWidth, DepthHeight, PixelFormats.Bgr32, bg, 3); median = new AForge.Imaging.Filters.Median(5); if (null == globalBWorker) { globalBWorker = new BackgroundWorker(); globalBWorker.DoWork += DoWorking; globalBWorker.RunWorkerAsync(); } } ////PlayerBitmap.WritePixels(new Int32Rect(0, 0, DepthWidth, DepthHeight), //// PlayerPixels, DepthWidth * ((PlayerBitmap.Format.BitsPerPixel + 7) / 8), 0); }
void KinectFaceNode_AllFrameReady(object sender, AllFramesReadyEventArgs e) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; colorImageFrame = e.OpenColorImageFrame(); depthImageFrame = e.OpenDepthImageFrame(); skeletonFrame = e.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { return; } if (first) { first = false; this.olddepth = depthImageFrame.Format; } else { if (this.olddepth != depthImageFrame.Format) { //Need a reset if (this.depthImage != null) { this.depthImage = null; } if (this.face != null) { this.face.Dispose(); this.face = null; } this.trackedSkeletons.Clear(); this.olddepth = depthImageFrame.Format; } } if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } if (face == null) { face = new FaceTracker(this.runtime.Runtime); } colorImageFrame.CopyPixelDataTo(this.colorImage); depthImageFrame.CopyPixelDataTo(this.depthImage); skeletonFrame.CopySkeletonDataTo(this.skeletonData); foreach (Skeleton skeleton in this.skeletonData) { if (skeleton.TrackingState == SkeletonTrackingState.Tracked || skeleton.TrackingState == SkeletonTrackingState.PositionOnly) { // We want keep a record of any skeleton, tracked or untracked. if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId)) { this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker()); } // Give each tracker the upated frame. SkeletonFaceTracker skeletonFaceTracker; if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker)) { skeletonFaceTracker.OnFrameReady(this.runtime.Runtime, colorImageFrame.Format, colorImage, depthImageFrame.Format, depthImage, skeleton); skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber; } } } this.RemoveOldTrackers(skeletonFrame.FrameNumber); colorImageFrame.Dispose(); depthImageFrame.Dispose(); skeletonFrame.Dispose(); this.FInvalidate = true; }
private void sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e) { //TODO Keep the previous frame image as well, //Compare both on a background process and save it to the worksheet //Convert x&y differences to millimeters according to depth data (distance) //and some trigonometry BitmapSource depthBmp = null; blobCount = 0; using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame != null) { blobCount = 0; depthBmp = depthFrame.SliceDepthImage((int)sliderMin.Value, (int)sliderMax.Value); Image <Bgr, Byte> openCVImg = new Image <Bgr, byte>(depthBmp.ToBitmap()); Image <Gray, byte> gray_image = openCVImg.Convert <Gray, byte>(); if (running) { wsheet.Cells[1, frameCount + 1].Value = "Frame " + frameCount; frameCount++; using (MemStorage stor = new MemStorage()) { //Find contours with no holes try CV_RETR_EXTERNAL to find holes Contour <System.Drawing.Point> contours = gray_image.FindContours( Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL, stor); //Conversion of depthPixels to skeletonPoints which contain all three dimensions in meters. //The conversion and copying is assumed to be costly but there are no single pixel to single point conversion I could find. depthFrame.CopyDepthImagePixelDataTo(depthPixels); //mapper.MapDepthFrameToSkeletonFrame(depthFormat, depthPixels, skeletonPoints); for (int i = 0; contours != null; contours = contours.HNext) { i++; if ((contours.Area > Math.Pow(sliderMinSize.Value, 2)) && (contours.Area < Math.Pow(sliderMaxSize.Value, 2))) { MCvBox2D box = contours.GetMinAreaRect(); //DrQ RED BOX AROUND BLOB openCVImg.Draw(box, new Bgr(System.Drawing.Color.Red), 2); blobCount++; int x = (int)box.center.X; int y = (int)box.center.Y; DepthImagePoint p = new DepthImagePoint(); p.X = x; p.Y = y; p.Depth = depthPixels[x + 640 * y].Depth; SkeletonPoint s = mapper.MapDepthPointToSkeletonPoint(depthFormat, p); //TODO Conversion from absolute coordinates to relative coordinates addCoordData(3 * blobCount - 1, frameCount, s.X, s.Y, s.Z); /*if (KinectSensor.IsKnownPoint(s)) * { * addCoordData(3 * blobCount - 1, frameCount, s.X, s.Y, s.Z); * }*/ } } } } this.outImg.Source = ImageHelpers.ToBitmapSource(openCVImg); txtBlobCount.Text = blobCount.ToString(); getNext().RunWorkerAsync(openCVImg); } } if (colorFrame != null) { colorFrame.CopyPixelDataTo(this.colorPixels); this.colorBitmap.WritePixels( new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight), this.colorPixels, this.colorBitmap.PixelWidth * sizeof(int), 0); } } }
/// <summary> /// Event handler for Kinect sensor's DepthFrameReady event /// </summary> /// <param name="sender">object sending the event</param> /// <param name="e">event arguments</param> private void SensorAllFramesReady(object sender, AllFramesReadyEventArgs e) { // in the middle of shutting down, or lingering events from previous sensor, do nothing here. if (null == this.sensorChooser || null == this.sensorChooser.Kinect || this.sensorChooser.Kinect != sender) { return; } try { using (var depthFrame = e.OpenDepthImageFrame()) { if (null != depthFrame) { this.backgroundRemovedColorStream.ProcessDepth(depthFrame.GetRawPixelData(), depthFrame.Timestamp); } } using (var colorFrame = e.OpenColorImageFrame()) { if (null != colorFrame) { this.backgroundRemovedColorStream.ProcessColor(colorFrame.GetRawPixelData(), colorFrame.Timestamp); } } using (var skeletonFrame = e.OpenSkeletonFrame()) { if (null != skeletonFrame) { skeletonFrame.CopySkeletonDataTo(this.skeletons); this.backgroundRemovedColorStream.ProcessSkeleton(this.skeletons, skeletonFrame.Timestamp); } } this.ChooseSkeleton(); } catch (InvalidOperationException) { // Ignore the exception. } }
private void Kinect_AllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; try { colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { return; } // Check for image format changes. The FaceTracker doesn't // deal with that so we need to reset. if (this.depthImageFormat != depthImageFrame.Format) { this.depthImage = null; this.depthImageFormat = depthImageFrame.Format; } if (this.colorImageFormat != colorImageFrame.Format) { this.colorImage = null; this.colorImageFormat = colorImageFrame.Format; } // Create any buffers to store copies of the data we work with if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } // Get the skeleton information if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } // TODO look into using the Timestamp on each frame var time = pipeline.GetCurrentTime(); var sharedColorImage = ImagePool.GetOrCreate(colorImageFrame.Width, colorImageFrame.Height, Imaging.PixelFormat.BGRX_32bpp); var sharedDepthImage = ImagePool.GetOrCreate(depthImageFrame.Width, depthImageFrame.Height, Imaging.PixelFormat.Gray_16bpp); colorImageFrame.CopyPixelDataTo(sharedColorImage.Resource.ImageData, (colorImageFrame.Width * colorImageFrame.Height * 4)); this.ColorImage.Post(sharedColorImage, time); //depthImageFrame.CopyPixelDataTo(sharedDepthImage.Resource.ImageData, (depthImageFrame.Width * depthImageFrame.Height * 2)); depthImageFrame.CopyPixelDataTo(sharedDepthImage.Resource.ImageData, depthImageFrame.PixelDataLength); this.DepthImage.Post(sharedDepthImage, time); skeletonFrame.CopySkeletonDataTo(this.skeletonData); this.Skeletons.Post(this.skeletonData.ToList(), time); } catch { // TODO catch a cold } }
void kinectRuntime_AllFrameReady(object sender, AllFramesReadyEventArgs e) { // If replaying, bypass this. if (replay != null && !replay.IsFinished) return; using (var cf = e.OpenColorImageFrame()) using (var df = e.OpenDepthImageFrame()) using (var sf = e.OpenSkeletonFrame()) { try { if (recorder != null && sf != null && df != null && cf != null) { recorder.Record(sf, df, cf); } } catch (ObjectDisposedException) { } if (cf != null) colorManager.Update(cf, !displayDebug); if (df != null) { depthFrameNumber = df.FrameNumber; depthManager.Update(df); } if (sf != null) { UpdateSkeletonDisplay(sf); if (handTracker != null && recogEngine != null) { var result = handTracker.Update(depthManager.PixelData, colorManager.PixelData, SkeletonUtil.FirstTrackedSkeleton(sf.GetSkeletons())); var gesture = recogEngine.Update(result); lock (inputServer) inputServer.Send(gesture); UpdateDisplay(result); textGestureEvent.Text = gesture; fpsCounter.LogFPS(); } } } }
/// <summary> /// Event handler for Kinect sensor's AllFramesReady event. /// </summary> /// <param name="sender">Object sending the event.</param> /// <param name="e">Event arguments.</param> private void SensorAllFramesReady(object sender, AllFramesReadyEventArgs e) { // Already shutting down, or lingering events from previous sensor: do nothing here. if (null == this.sensor || this.sensor != sender) { return; } try { if (this.IsTracked) { using (var depthFrame = e.OpenDepthImageFrame()) { if (null != depthFrame) { // Process depth data for background removal. this.backgroundRemovedColorStream.ProcessDepth( depthFrame.GetRawPixelData(), depthFrame.Timestamp); } } using (var colorFrame = e.OpenColorImageFrame()) { if (null != colorFrame) { // Process color data for background removal. this.backgroundRemovedColorStream.ProcessColor( colorFrame.GetRawPixelData(), colorFrame.Timestamp); } } using (var skeletonFrame = e.OpenSkeletonFrame()) { if (null != skeletonFrame) { // Save skeleton frame data for subsequent processing. this.CopyDataFromSkeletonFrame(skeletonFrame); // Locate the most recent data in which this user was fully tracked. bool isUserPresent = this.UpdateTrackedSkeletonsArray(); // If we have an array in which this user is fully tracked, // process the skeleton data for background removal. if (isUserPresent && null != this.skeletonsTracked) { this.backgroundRemovedColorStream.ProcessSkeleton( this.skeletonsTracked, skeletonFrame.Timestamp); } } } } } catch (InvalidOperationException) { // Ignore the exception. } }
/// <summary> /// Event handler for Kinect sensor's SkeletonFrameReady event /// </summary> /// <param name="sender">object sending the event</param> /// <param name="e">event arguments</param> private void SensorAllFramesReady(object sender, AllFramesReadyEventArgs e) { if (shuttingDown) { return; } KinectSensor sensor = (KinectSensor)sender; ColorImageFrame colorFrame = e.OpenColorImageFrame(); if (colorFrame == null) return; if (colorPixelData[sensor] == null) colorPixelData[sensor] = new byte[colorFrame.PixelDataLength]; colorFrame.CopyPixelDataTo(colorPixelData[sensor]); colorFrame = null; DepthImageFrame depthFrame = e.OpenDepthImageFrame(); if (depthFrame == null) return; if (depthPixelData[sensor] == null) depthPixelData[sensor] = new short[depthFrame.PixelDataLength]; depthFrame.CopyPixelDataTo(depthPixelData[sensor]); depthFrame = null; SkeletonFrame skeletonFrame = e.OpenSkeletonFrame(); if (skeletonFrame == null) return; skeletonFrame.CopySkeletonDataTo(skeletons[sensor]); skeletonFrame = null; SensorFrameHelper(sensor, true); }
void FramesReady(object sender, AllFramesReadyEventArgs e) { DepthImageFrame imageFrame = e.OpenDepthImageFrame(); ColorImageFrame rgbFrame = e.OpenColorImageFrame(); if (imageFrame != null && rgbFrame != null) { int targetPixelDist = int.MaxValue; int[] targetPixelPos = { imageFrame.Width / 2, imageFrame.Height / 2 }; short[] pixelData = new short[imageFrame.PixelDataLength]; imageFrame.CopyPixelDataTo(pixelData); Parallel.For(0, imageFrame.Height, new ParallelOptions { MaxDegreeOfParallelism = 16 }, y => //int x = 0; x < imageFrame.Width; x++) { //ColorImagePoint temp = imageFrame.MapToColorImagePoint(50, y, ColorImageFormat.RgbResolution640x480Fps30); for (int x = 0; x < imageFrame.Width; x++) { if ((ushort)((pixelData[x + y * 640]) >> 3) < maxDist && (ushort)((pixelData[x + y * 640]) >> 3) > minDist) { //Check if pixel is closest one if ((ushort)((pixelData[x + y * 640]) >> 3) < targetPixelDist) { targetPixelDist = (ushort)((pixelData[x + y * 640]) >> 3); //When you think you're going to use a really long and ugly value once but it turns out you use it a bunch targetPixelPos[0] = x; targetPixelPos[1] = y; } } } }); pictureBox1.Image = ImageToBitmap(rgbFrame); Graphics drawStats = Graphics.FromImage(pictureBox1.Image); drawStats.FillEllipse(Brushes.Red, targetPixelPos[0], targetPixelPos[1], 10, 10); //Start figuring out where to move it if on a frame that is multiple of 4 frameCount++; if (frameCount >= 4) { while (port.BytesToRead > 0) { port.ReadByte(); //Throw the bytes away } frameCount = 0; if (targetPixelPos[0] != imageFrame.Width / 2 && targetPixelPos[1] != imageFrame.Height / 2) { //Detected something numberOfTimesDetectedSomething++; } else //No target found { numberOfTimesDetectedNothing++; } if (numberOfTimesDetectedSomething >= 3) { if (numberOfTimesDetectedNothing > 3) { numberOfTimesDetectedNothing = 0; } if (targetPixelPos[1] < (imageFrame.Height / 2) - threshold) { yawLabel.Text = "Up"; port.WriteLine("U"); port.ReadLine(); } else if (targetPixelPos[1] > (imageFrame.Height / 2) + threshold) { yawLabel.Text = "Down"; port.WriteLine("D"); port.ReadLine(); } else { yawLabel.Text = "Target yaw"; } //Pitch if (targetPixelPos[0] < (imageFrame.Width / 2) - threshold) { pitchLabel.Text = "Left"; port.WriteLine("R"); port.ReadLine(); } else if (targetPixelPos[0] > (imageFrame.Width / 2) + threshold) { pitchLabel.Text = "Right"; port.WriteLine("L"); port.ReadLine(); } else { pitchLabel.Text = "Target pitch"; } } /*if(numberOfTimesDetectedNothing >= 3) * { * if(numberOfTimesDetectedSomething > 3) numberOfTimesDetectedSomething = 0; //If statement fixes it, no idea why * //Start scanning for drones * //Set yaw correctly * port.WriteLine("P"); //P means set for scanning edit: I should have used S * port.ReadLine(); * port.WriteLine("L"); * port.ReadLine(); * }*/ } } }
void sensor_allFramesReady(object sender, AllFramesReadyEventArgs e) { if (windowClosing) { return; } int value; if (int.TryParse(this.textBox_init.Text, out value)) { initFrames = value; } if (fpsEnd == 1) { //FPS Suggestion. Bei niedrigen Frameraten werden empfangene Frames übersprungen (nicht angezeigt) Int16 fps = Convert.ToInt16(this.dropDown_fps.Text); switch (fps) { case 30: fpsEnd = 1; break; case 15: fpsEnd = 2; break; case 10: fpsEnd = 3; break; case 5: fpsEnd = 6; break; case 1: fpsEnd = 30; break; } using (ColorImageFrame colorFrame = e.OpenColorImageFrame()) { if (colorFrame != null) { // Kinect Color Frame to Bitmap tempColorFrame = ColorImageFrameToBitmap(colorFrame); this.pictureBox_colorPic.BackgroundImage = tempColorFrame; // this.pictureBox_colorPic.Image = new Bitmap(tempColorFrame, this.pictureBox_colorPic.Width, this.pictureBox_colorPic.Height); // Too slow ifnot scaled down by 10x //this.pictureBox_colorPic.BackgroundImage = BitmapManipulator.ConvertImageToHeatmapBitMap(BitmapManipulator.ScaleBitmap(tempColorFrame, 0.1, 0.1, System.Drawing.Drawing2D.InterpolationMode.Default), Heatmap, false); } } using (DepthImageFrame depthFrame = e.OpenDepthImageFrame()) { if (depthFrame != null) { // Kinect Depth Frame to Bitmap Bitmap tempDepthFrame = DepthImageFrameToBitmap(depthFrame); this.pictureBox_depthPic.BackgroundImage = tempDepthFrame; // this.pictureBox_depthPic.Image = new Bitmap(tempDepthFrame, this.pictureBox_depthPic.Width, this.pictureBox_depthPic.Height); objectsFound = 0; depthBmp = ImageHelpers.SliceDepthImage(depthFrame, (int)trackBarDepthMinDistance.Value, (int)trackBarDepthMaxDistance.Value); Image<Bgr, Byte> openCVImg = new Image<Bgr, byte>(depthBmp); Image<Gray, byte> gray_image = openCVImg.Convert<Gray, byte>(); using (MemStorage stor = new MemStorage()) { //Find contours with no holes try CV_RETR_EXTERNAL to find holes Contour<System.Drawing.Point> contours = gray_image.FindContours( Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE, Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL, stor); for (int i = 0; contours != null; contours = contours.HNext) { i++; if ((contours.Area > Math.Pow(trackBarObjectMinSize.Value, 2)) && (contours.Area < Math.Pow(trackBarObjectMaxSize.Value, 2))) { MCvBox2D box = contours.GetMinAreaRect(); openCVImg.Draw(box, new Bgr(System.Drawing.Color.Red), 2); objectsFound++; } } } pictureBox_depthPicSmoothed.BackgroundImage = openCVImg.Bitmap; richTextBoxObjectFound.Text = objectsFound.ToString(); // The Smoothed Image will apply both a filter and a weighted moving average over the // depth data from the Kinect (depending on UI selections) //pictureBox_depthPicSmoothed.BackgroundImage = CreateSmoothImageFromDepthArray(tempDepthFrame); // pictureBox_depthPicSmoothed.Image = CreateSmoothImageFromDepthArray(tempDepthFrame); // Create a pseudo color of the depth map // Too slow for 30 frames per second if not scaled down by 10x //this.pictureBox_depthPicSmoothed.BackgroundImage = BitmapManipulator.ConvertImageToHeatmapBitMap(BitmapManipulator.ScaleBitmap(tempDepthFrame, 0.1, 0.1, System.Drawing.Drawing2D.InterpolationMode.Default), Heatmap, false); // Create a byte arrya of the depth data //ConvertDepthFrameData(sender, depthFrame); DepthImagePixel[] depth = depthFrame.GetRawPixelData(); pictureBox_colorPic.BackgroundImage = LightBuzz.Vitruvius.WinForms.DepthExtensions.ToBitmap(depthFrame, PixelFormat.Format32bppRgb, LightBuzz.Vitruvius.WinForms.DepthImageMode.Colors); } } using (SkeletonFrame skelFrame = e.OpenSkeletonFrame()) { if (skelFrame != null) { Image tempSkeletonFrame = new Bitmap(this.pictureBox_skeleton.Width, this.pictureBox_skeleton.Height); // make the background black if there is no image this.pictureBox_skeleton.BackColor = Color.Black; this.pictureBox_skeleton.BackgroundImage = null; if (checkBox_colorCam.Checked) { tempSkeletonFrame = new Bitmap(tempColorFrame); } if (this.checkBoxShowSkeleton.Checked) { Skeleton[] skeletons = new Skeleton[skelFrame.SkeletonArrayLength]; skelFrame.CopySkeletonDataTo(skeletons); if (skeletons.Length != 0) { foreach (Skeleton skel in skeletons) { if (skel.TrackingState == SkeletonTrackingState.Tracked) { //Zeichne Skelett DrawSkeletons(tempSkeletonFrame, skel); if (skel != null) { double height = LightBuzz.Vitruvius.SkeletonExtensions.Height(skel); } if (NGEFile != null) { if (NGEFile.isRecording == true && NGEFile.isInitializing == true) { NGEFile.Entry(skel); if (NGEFile.intializingCounter > initFrames) { NGEFile.startWritingEntry(); } } if (NGEFile.isRecording == true && NGEFile.isInitializing == false) { NGEFile.Motion(skel); this.textBox_sensorStatus.Text = "Record"; this.textBox_sensorStatus.BackColor = Color.Green; } } } } } this.pictureBox_skeleton.BackgroundImage = tempSkeletonFrame; // this.pictureBox_skeleton.Image = tempSkeletonFrame; //this.pictureBox_skeleton.Image = new Bitmap(tempSkeletonFrame, this.pictureBox_skeleton.Width, this.pictureBox_skeleton.Height); } } } } else { fpsEnd -= 1; } UpdateFps(); }
private void AllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs) { ColorImageFrame colorImageFrame = null; DepthImageFrame depthImageFrame = null; SkeletonFrame skeletonFrame = null; try { colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null) { return; } // Check for changes in any of the data this function is receiving // and reset things appropriately. if (this.depthImageFormat != depthImageFrame.Format) { this.DestroyFaceTracker(); this.depthImage = null; this.depthImageFormat = depthImageFrame.Format; } if (this.colorImageFormat != colorImageFrame.Format) { this.DestroyFaceTracker(); this.colorImage = null; this.colorImageFormat = colorImageFrame.Format; this.colorImageWritableBitmap = null; this.ColorImage.Source = null; this.theMaterial.Brush = null; } if (this.skeletonData != null && this.skeletonData.Length != skeletonFrame.SkeletonArrayLength) { this.skeletonData = null; } // Create any buffers to store copies of the data we work with if (this.depthImage == null) { this.depthImage = new short[depthImageFrame.PixelDataLength]; } if (this.colorImage == null) { this.colorImage = new byte[colorImageFrame.PixelDataLength]; } if (this.colorImageWritableBitmap == null) { this.colorImageWritableBitmap = new WriteableBitmap( colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null); this.ColorImage.Source = this.colorImageWritableBitmap; this.theMaterial.Brush = new ImageBrush(this.colorImageWritableBitmap) { ViewportUnits = BrushMappingMode.Absolute }; } if (this.skeletonData == null) { this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength]; } // Copy data received in this event to our buffers. colorImageFrame.CopyPixelDataTo(this.colorImage); depthImageFrame.CopyPixelDataTo(this.depthImage); skeletonFrame.CopySkeletonDataTo(this.skeletonData); this.colorImageWritableBitmap.WritePixels( new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height), this.colorImage, colorImageFrame.Width * Bgr32BytesPerPixel, 0); // Find a skeleton to track. // First see if our old one is good. // When a skeleton is in PositionOnly tracking state, don't pick a new one // as it may become fully tracked again. Skeleton skeletonOfInterest = this.skeletonData.FirstOrDefault( skeleton => skeleton.TrackingId == this.trackingId && skeleton.TrackingState != SkeletonTrackingState.NotTracked); if (skeletonOfInterest == null) { // Old one wasn't around. Find any skeleton that is being tracked and use it. skeletonOfInterest = this.skeletonData.FirstOrDefault( skeleton => skeleton.TrackingState == SkeletonTrackingState.Tracked); if (skeletonOfInterest != null) { // This may be a different person so reset the tracker which // could have tuned itself to the previous person. if (this.faceTracker != null) { this.faceTracker.ResetTracking(); } this.trackingId = skeletonOfInterest.TrackingId; } } if (skeletonOfInterest != null && skeletonOfInterest.TrackingState == SkeletonTrackingState.Tracked) { if (this.faceTracker == null) { try { this.faceTracker = new FaceTracker(this.Kinect); } catch (InvalidOperationException) { // During some shutdown scenarios the FaceTracker // is unable to be instantiated. Catch that exception // and don't track a face. Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException"); this.faceTracker = null; } } if (this.faceTracker != null) { FaceTrackFrame faceTrackFrame = this.faceTracker.Track( this.colorImageFormat, this.colorImage, this.depthImageFormat, this.depthImage, skeletonOfInterest); if (faceTrackFrame.TrackSuccessful) { if (!visited) { visited = true; //counter.Text = "60 seconds"; aTimer.Interval = 1000; aTimer.Tick += new EventHandler(aTimer_Tick); aTimer.Start(); } if (saveModel) { saveDepthImagebmp(depthImageFrame); saveColorImage(colorImageFrame.Width, colorImageFrame.Height, (colorImageFrame.Width * Bgr32BytesPerPixel)); saveFaceModel(); } } } } else { this.trackingId = -1; } } finally { if (colorImageFrame != null) { colorImageFrame.Dispose(); } if (depthImageFrame != null) { depthImageFrame.Dispose(); } if (skeletonFrame != null) { skeletonFrame.Dispose(); } } }