/// <summary> 手のデータを更新する </summary> private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // データを初期化する CanvasFaceParts.Children.Clear(); // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } GetFingerData(hand, PXCMHandData.JointType.JOINT_MIDDLE_TIP); DetectTap(hand); } }
/// <summary> 手のデータを更新する </summary> private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // データを初期化する PartsCanvas.Children.Clear(); // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } //Console.WriteLine(hand.QueryUniqueId()); int side = (int)hand.QueryBodySide(); if (side > 0) { side2id[side - 1] = hand.QueryUniqueId(); } GetFingerData(hand, PXCMHandData.JointType.JOINT_MIDDLE_TIP); } }
// Update is called once per frame void Update() { if (sm != null) { /* Wait until any frame data is available */ if (sm.AcquireFrame(false, 0) == pxcmStatus.PXCM_STATUS_NO_ERROR) { /* Retrieve latest extremity data */ if (hand_data.Update() == pxcmStatus.PXCM_STATUS_NO_ERROR) { ; } { TrackExtremity(hand_data); if (hand_data.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (handData.QueryExtremityPoint(PXCMHandData.ExtremityType.EXTREMITY_LEFTMOST, out extremityPoint) == pxcmStatus.PXCM_STATUS_NO_ERROR) { Debug.Log("LeftMost Extremity Position = " + extremityPoint.pointWorld); } } } /* Now, release the current frame so we can process the next frame */ sm.ReleaseFrame(); } } }
// Update is called once per frame void Update() { if (sm != null) { /* Wait until any frame data is available */ if (sm.AcquireFrame(false, 0) == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (hand_data.QueryNumberOfHands() == 0) { calibText.text = ""; } /* Retrieve latest hand data, only update slingshot if ready */ if (hand_data.Update() == pxcmStatus.PXCM_STATUS_NO_ERROR) { ; } { TrackJoints(hand_data); pDrag.UpdateDrag(isPinching); } /* Now, release the current frame so we can process the next frame */ sm.ReleaseFrame(); } } }
void Update() { if (mRS.AcquireFrame(false) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { mHand.Update(); getCursorPositions(); mRS.ReleaseFrame(); } }
// 手のデータを更新する private void UpdateHandFrame() { handData.Update(); // 画像の初期化 ImageHand1.Source = ImageHand2.Source = null; // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 手の画像を取得する PXCMImage image; sts = hand.QuerySegmentationImage(out image); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // マスク画像を取得する PXCMImage.ImageData data; sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out data); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // マスク画像のサイズはDepthに依存 // 手は2つまで var info = image.QueryInfo(); var bitmap = BitmapSource.Create(info.width, info.height, 96, 96, PixelFormats.Gray8, null, data.planes[0], data.pitches[0] * info.height, data.pitches[0]); if (i == 0) { ImageHand1.Source = bitmap; } else { ImageHand2.Source = bitmap; } image.ReleaseAccess(data); } }
// Update is called once per frame void Update() { /* Make sure PXCMSenseManager Instance is Initialized */ if (SenseToolkitManager.Instance.SenseManager == null) { return; } /* Wait until any frame data is available true(aligned) false(unaligned) */ if (SenseToolkitManager.Instance.SenseManager.AcquireFrame(false, 0) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } /* Retrieve am instance of hand tracking Module */ handAnalyzer = SenseToolkitManager.Instance.SenseManager.QueryHand(); if (handAnalyzer != null) { /* Retrieve an instance of hand tracking Data */ PXCMHandData _outputData = handAnalyzer.CreateOutput(); if (_outputData != null) { _outputData.Update(); //Retrieve joint data, gesture recognition data and alert notification data //refer to next section //AcquireFrame /* Retrieve Gesture Data */ PXCMHandData.GestureData _gestureData; for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++) { if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { //Display the gestures: explained in rendering the frame section if ((_gestureData.name == "spreadfingers") || (_gestureData.name == "tap") || (_gestureData.name == "swipe")) { CursorController.isHandClicked = false; } else if ((_gestureData.name == "fist") || (_gestureData.name == "full_pinch") || (_gestureData.name == "thumb_down") || (_gestureData.name == "thumb_up") || (_gestureData.name == "two_fingers_pinch_open") || (_gestureData.name == "v_sign")) { CursorController.isHandClicked = true; } else if (_gestureData.name == "wave") { transform.position = new Vector3(0, 0, 0); } } } } } /* Realease the frame to process the next frame */ SenseToolkitManager.Instance.SenseManager.ReleaseFrame(); }
// 手のデータを更新する private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // データを初期化する CanvasFaceParts.Children.Clear(); // 点に色を付ける Brush[] colors = new Brush[] { Brushes.Blue, Brushes.Green, Brushes.Red, Brushes.Yellow, Brushes.Purple, Brushes.LightBlue, }; // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 指の関節を列挙する for (int j = 0; j < PXCMHandData.NUMBER_OF_EXTREMITIES; j++) { PXCMHandData.ExtremityData extremityData; sts = hand.QueryExtremityPoint((PXCMHandData.ExtremityType)j, out extremityData); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } AddEllipse(CanvasFaceParts, new Point(extremityData.pointImage.x, extremityData.pointImage.y), 10, colors[j], -1); } } }
public override void Work(Graphics g) { data = module.CreateOutput(); data.Update(); PXCMHandData.IHand hands = null; data.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, 0, out hands); if (hands == null) { return; } PXCMRectI32 rect = hands.QueryBoundingBoxImage(); Rectangle rectangle = new Rectangle(rect.x, rect.y, rect.w, rect.h); // Convert to Rectangle g.DrawRectangle(pen, rectangle); // Draw }
public void Loop(LoopObjects loopObjects) { _loopObjects = loopObjects; var handModule = _camera.Manager.QueryHand(); if (handModule == null) { return; } _handData = handModule.CreateOutput(); _handData.Update(); TrackHand(_camera.LeftHand, PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS); TrackHand(_camera.RightHand, PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS); }
// 手のデータを更新する private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // データを初期化する CanvasFaceParts.Children.Clear(); // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 指の関節を列挙する for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++) { // 指のデータを取得する PXCMHandData.JointData jointData; sts = hand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // Depth座標系をカラー座標系に変換する var depthPoint = new PXCMPoint3DF32[1]; var colorPoint = new PXCMPointF32[1]; depthPoint[0].x = jointData.positionImage.x; depthPoint[0].y = jointData.positionImage.y; depthPoint[0].z = jointData.positionWorld.z * 1000; projection.MapDepthToColor(depthPoint, colorPoint); AddEllipse(CanvasFaceParts, new Point(colorPoint[0].x, colorPoint[0].y), 5, Brushes.Green); } } }
public override void Work(Graphics g) { // Retrieve gesture data hand = senseManager.QueryHand(); face = senseManager.QueryFace(); // if (hand != null) { // Retrieve the most recent processed data handData = hand.CreateOutput(); handData.Update(); handWaving = handData.IsGestureFired("wave", out gestureData); } if (face != null) { faceData = face.CreateOutput(); faceData.Update(); //surching faces Int32 nfaces = faceData.QueryNumberOfDetectedFaces(); for (Int32 i = 0; i < nfaces; i++) { // Retrieve the data instance PXCMFaceData.Face faceI = faceData.QueryFaceByIndex(i); PXCMFaceData.ExpressionsData edata = faceI.QueryExpressions(); if (edata != null) { edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out score); if (score.intensity >= 25) { smiling = true; } else { smiling = false; } Console.WriteLine(i + ": " + score.intensity); } } faceData.Dispose(); } }
// 手のデータを更新する private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // データを初期化する CanvasFaceParts.Children.Clear(); // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 指の関節を列挙する for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++) { PXCMHandData.JointData jointData; sts = hand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } AddEllipse(CanvasFaceParts, new Point(jointData.positionImage.x, jointData.positionImage.y), 5, Brushes.Green); } } }
void Update() { if (gestureCooldown > 0) { gestureCooldown -= Time.deltaTime; } if (psm == null) { return; } if (psm.AcquireFrame(true) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } drawCameraVision(); handData.Update(); if (currentEffect == 2) { PXCMHandData.JointData[,] nodes = getHandInfo(); followJoints(nodes); } else if (currentEffect == 3) { // point cloud createDepthMesh(); } moveHands(); handleGestures(); // Release the frame psm.ReleaseFrame(); }
private void ProcessingThread() { // Start AcquireFrame/ReleaseFrame loop while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMCapture.Sample sample = senseManager.QuerySample(); Bitmap colorBitmap; PXCMImage.ImageData colorData; // Get color image data sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData); colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); // Retrieve gesture data hand = senseManager.QueryHand(); if (hand != null) { // Retrieve the most recent processed data handData = hand.CreateOutput(); handData.Update(); handWaving = handData.IsGestureFired("v_sign", out gestureData); } // Update the user interface UpdateUI(colorBitmap); // Release the frame if (handData != null) { handData.Dispose(); } colorBitmap.Dispose(); sample.color.ReleaseAccess(colorData); senseManager.ReleaseFrame(); } }
// Update is called once per frame void Update() { handData = SenseToolkitManager.Instance.HandDataOutput; if (handData != null) { handData.Update(); for (int i = 0; i < handData.QueryFiredGesturesNumber(); i++) { if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { DisplayGestures(gestureData); } } PXCMHandData.AlertData alertData; for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++) { if (handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { ProcessAlerts(alertData); } } } }
// 手のデータを更新する private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // ピクセルデータを初期化する Array.Clear(imageBuffer, 0, imageBuffer.Length); CanvasFaceParts.Children.Clear(); // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 手の画像を取得する PXCMImage image; sts = hand.QuerySegmentationImage(out image); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // マスク画像を取得する PXCMImage.ImageData data; sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out data); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 手の左右を取得する var side = (int)hand.QueryBodySide(); // 手の開閉度(0-100)を取得する var openness = hand.QueryOpenness(); // マスク画像のサイズはDepthに依存 // 手は2つまで var info = image.QueryInfo(); // マスク画像をバイト列に変換する var buffer = data.ToByteArray(0, data.pitches[0] * info.height); for (int j = 0; j < info.height * info.width; ++j) { if (buffer[j] != 0) { var index = j * BYTE_PER_PIXEL; // 手の左右および手の開閉度で色合いを決める // 左手=1:0-127の範囲 // 右手=2:0-254の範囲 var value = (byte)((side * 127) * (openness / 100.0f)); imageBuffer[index + 0] = value; imageBuffer[index + 1] = value; imageBuffer[index + 2] = value; } } image.ReleaseAccess(data); // 手の重心を表示する var center = hand.QueryMassCenterImage(); var ellipse = new Ellipse() { Width = 5, Height = 5, Fill = Brushes.Blue, }; Canvas.SetLeft(ellipse, center.x); Canvas.SetTop(ellipse, center.y); CanvasFaceParts.Children.Add(ellipse); // 手の範囲を表示する var boundingbox = hand.QueryBoundingBoxImage(); var rectangle = new Rectangle() { Width = boundingbox.w, Height = boundingbox.h, Stroke = Brushes.Red, StrokeThickness = 2, }; Canvas.SetLeft(rectangle, boundingbox.x); Canvas.SetTop(rectangle, boundingbox.y); CanvasFaceParts.Children.Add(rectangle); } // ピクセルデータを更新する imageBitmap.WritePixels(imageRect, imageBuffer, DEPTH_WIDTH * BYTE_PER_PIXEL, 0); }
public static void start(IoT_RealSense_Surfing frm) { keepLooping = true; iot_form = frm; InitializeMqqtClient(); PXCMSession session = PXCMSession.CreateInstance(); // Querying the SDK version Console.WriteLine(frm.comboTarget.SelectedText); if (session != null) { // Optional steps to send feedback to Intel Corporation to understand how often each SDK sample is used. PXCMMetadata md = session.QueryInstance <PXCMMetadata>(); if (md != null) { string sample_name = "Emotion Viewer CS"; md.AttachBuffer(1297303632, System.Text.Encoding.Unicode.GetBytes(sample_name)); } //Application.Run(new MainForm(session)); //session.Dispose(); } //PXCMSession.ImplVersion version = session.QueryVersion(); //Console.WriteLine("RealSense SDK Version {0}.{1}", version.major, version.minor); session.CreateImpl <PXCMRotation>(out rotationHelper); // Creating the SenseManager PXCMSenseManager senseManager = session.CreateSenseManager(); if (senseManager == null) { Console.WriteLine("Failed to create the SenseManager object."); return; } // Enabling Emotion Module pxcmStatus enablingModuleStatus = senseManager.EnableEmotion(); if (enablingModuleStatus != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("Failed to enable the Emotion Module"); return; } // Getting the instance of the Emotion Module PXCMEmotion emotionModule = senseManager.QueryEmotion(); if (emotionModule == null) { Console.WriteLine("Failed to query the emotion module"); return; } // Initializing the camera if (senseManager.Init() < pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("Failed to initialize the SenseManager"); return; } // Enabling the Hand module pxcmStatus enablingModuleStatus1 = senseManager.EnableHand("Hand Module"); if (enablingModuleStatus1 != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("Failed to enable the Hand Module"); return; } // Getting the instance of the Hand Module PXCMHandModule handModule = senseManager.QueryHand(); if (handModule == null) { Console.WriteLine("Failed to get the HandModule object."); return; } // Creating an active configuration PXCMHandConfiguration handConfiguration = handModule.CreateActiveConfiguration(); if (handConfiguration == null) { Console.WriteLine("Failed to create the HandConfiguration object."); return; } // Listing the available gestures int supportedGesturesCount = handConfiguration.QueryGesturesTotalNumber(); if (supportedGesturesCount > 0) { Console.WriteLine("Supported gestures:"); for (int i = 0; i < supportedGesturesCount; i++) { string gestureName = string.Empty; if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) == pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("\t" + gestureName); } } } // Enabling some gestures String[] enabledGestures = { GESTURE_CLICK, GESTURE_VSIGN, GESTURE_FIST, GESTURE_SPREADFINGERS }; foreach (String gesture in enabledGestures) { if (!handConfiguration.IsGestureEnabled(gesture)) { handConfiguration.EnableGesture(gesture); } } handConfiguration.ApplyChanges(); // Creating a data output object PXCMHandData handData = handModule.CreateOutput(); if (handData == null) { Console.WriteLine("Failed to create the HandData object."); return; } // Initializing the SenseManager if (senseManager.Init() != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine(senseManager.Init()); return; } // Looping to query the hands information while (keepLooping) { // Acquiring a frame if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR) { break; } // Updating the hand data if (handData != null) { handData.Update(); } //ProcessHands(handData); ProcessGestures(handData); ProcessEmotions(emotionModule); // Releasing the acquired frame senseManager.ReleaseFrame(); /* using another frame to process different stuff? may be... * // Acquiring a frame * if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR) * { * break; * } * * // Processing Emotions * ProcessEmotions(emotionModule); * * // Releasing the acquired frame * senseManager.ReleaseFrame();*/ } // Releasing resources if (handData != null) { handData.Dispose(); } if (handConfiguration != null) { handConfiguration.Dispose(); } rotationHelper.Dispose(); senseManager.Close(); senseManager.Dispose(); session.Dispose(); client.Disconnect(); }
void Update() { //Dynamically Pause/Enable Modules int numberOfEnabledModules = 0; foreach (var option in _senseOptions) { if (option.RefCounter == 0 && option.Enabled) { if (option.ModuleCUID > 0) { SenseManager.PauseModule(option.ModuleCUID, true); } option.Enabled = false; } else if (option.RefCounter > 0 && !option.Enabled) { if (!option.Initialized) { OnDisable(); OnEnable(); Start(); } if (option.ModuleCUID > 0) { SenseManager.PauseModule(option.ModuleCUID, false); } option.Enabled = true; } if (option.Enabled) { numberOfEnabledModules++; } } //Update Speech commands if changed if (_speechCommandsChanged) { UpdateSpeechCommands(); SpeechManager.Reset(); } // Every frame update all the data if (Initialized && numberOfEnabledModules > 0) { _sts = SenseManager.AcquireFrame(true, 0); if (_sts == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled) { if (ImageRgbOutput != null) { ImageRgbOutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.color != null) { ImageRgbOutput = _captureSample.color; ImageRgbOutput.AddRef(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled || _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled) { if (ImageDepthOutput != null) { ImageDepthOutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.depth != null) { ImageDepthOutput = _captureSample.depth; ImageDepthOutput.AddRef(); /* GZ * if (!_isInitBlob) * { * PXCMImage.ImageInfo info = ImageDepthOutput.QueryInfo(); * BlobExtractor.Init(info); * BlobExtractor.SetMaxBlobs(MaxBlobsToDetect); * _isInitBlob = true; * } */ if (PointCloud == null) { PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled) { if (PointCloud == null) { PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } _sts = Projection.QueryVertices(ImageDepthOutput, PointCloud); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.UVMap).Enabled) { if (UvMap == null) { UvMap = new PXCMPointF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } Projection.QueryUVMap(ImageDepthOutput, UvMap); } } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled) { if (ImageIROutput != null) { ImageIROutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.ir != null) { ImageIROutput = _captureSample.ir; ImageIROutput.AddRef(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled) { if (Image3DSegmentationOutput != null) { Image3DSegmentationOutput.Dispose(); } PXCM3DSeg seg = SenseManager.Query3DSeg(); if (seg != null) { Image3DSegmentationOutput = seg.AcquireSegmentedImage(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled) { FaceModuleOutput.Update(); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled) { HandDataOutput.Update(); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled) { BlobDataOutput.Update(); } _captureSample = null; SenseManager.ReleaseFrame(); } //Speech if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled) { SpeechManager.QueryRecognizedCommands(out SpeechOutput); } } }
// Update is called once per frame void Update() { /* Make sure SenseManager Instance is valid */ if (sm == null) { return; } /* Wait until any frame data is available */ if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } /* Retrieve hand tracking Module Instance */ handAnalyzer = sm.QueryHand(); if (handAnalyzer != null) { /* Retrieve hand tracking Data */ PXCMHandData _handData = handAnalyzer.CreateOutput(); if (_handData != null) { _handData.Update(); /* Retrieve Gesture Data to manipulate GUIText */ PXCMHandData.GestureData gestureData; for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++) { if (_handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { DisplayGestures(gestureData); } } /* Retrieve Alert Data to manipulate GUIText */ PXCMHandData.AlertData alertData; for (int i = 0; i < _handData.QueryFiredAlertsNumber(); i++) { if (_handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { ProcessAlerts(alertData); } } /* Retrieve all joint Data */ for (int i = 0; i < _handData.QueryNumberOfHands(); i++) { PXCMHandData.IHand _iHand; if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR) { for (int j = 0; j < MaxJoints; j++) { if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i][j]) != pxcmStatus.PXCM_STATUS_NO_ERROR) { jointData[i][j] = null; } } if (!handList.ContainsKey(_iHand.QueryUniqueId())) { handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide()); } } } /* Smoothen and Display the Data - Joints and Bones*/ DisplayJoints(); } handAnalyzer.Dispose(); } sm.ReleaseFrame(); RotateCam(); }
// Update is called once per frame void Update() { if (session == null) { return; } // For accessing hand data handAnalyzer = session.QueryHand(); faceAnalyzer = session.QueryFace(); if (handAnalyzer != null) { PXCMHandData handData = handAnalyzer.CreateOutput(); if (handData != null) { handData.Update(); PXCMHandData.IHand IHAND; // Ihand instance for accessing future data // Int32 IhandData; // for QueryOpenness Value // PXCMPoint3DF32 location; // Stores hand tracking position //Fills IHAND with information to later be grabbed and used for tracking + openness handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, 0, out IHAND); // If there is data in Ihand if (IHAND != null) { // Debug.DrawLine(transform.position, hit.point, Color.red); // Inits hand tracking from the center of the hand. // location = IHAND.QueryMassCenterWorld(); // if (mCurrentDart != null) // { // Vector3 locationUnity = new Vector3(location.x, location.y, location.z); // mCurrentDart.transform.localPosition = locationUnity * RSScale; // } } } handAnalyzer.Dispose(); session.ReleaseFrame(); } if (faceAnalyzer != null) { PXCMFaceData facedata = faceAnalyzer.CreateOutput(); if (facedata != null) { Int32 nfaces = facedata.QueryNumberOfDetectedFaces(); for (Int32 i = 0; i < nfaces; i++) { // Retrieve the face landmark data instance PXCMFaceData.Face face = facedata.QueryFaceByIndex(i); PXCMFaceData.PoseData pdata = face.QueryPose(); // retrieve the pose information PXCMFaceData.PoseEulerAngles angles; pdata.QueryPoseAngles(out angles); Debug.Log("Eular Angles yaw : " + angles.yaw); Debug.Log("Eular Angles pitch: " + angles.pitch); Debug.Log("Eular Angles Roll: " + angles.roll); angles.pitch = gameObject.transform.rotation.z; angles.yaw = gameObject.transform.rotation.y; } // device is a PXCMCapture.Device instance } } }
// 手のデータを更新する private void UpdateHandFrame() { // 手のデータを更新する handData.Update(); // ピクセルデータを初期化する Array.Clear(imageBuffer, 0, imageBuffer.Length); // 検出した手の数を取得する var numOfHands = handData.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { // 手を取得する PXCMHandData.IHand hand; var sts = handData.QueryHandData( PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // 手の画像を取得する PXCMImage image; sts = hand.QuerySegmentationImage(out image); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // マスク画像を取得する PXCMImage.ImageData data; sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out data); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // マスク画像のサイズはDepthに依存 // 手は2つまで var info = image.QueryInfo(); // マスク画像をバイト列に変換する var buffer = data.ToByteArray(0, data.pitches[0] * info.height); for (int j = 0; j < info.height * info.width; ++j) { if (buffer[j] != 0) { var index = j * BYTE_PER_PIXEL; // 手のインデックスで色を決める // ID=0:127 // ID=1:254 var value = (byte)((i + 1) * 127); imageBuffer[index + 0] = value; imageBuffer[index + 1] = value; imageBuffer[index + 2] = value; } } image.ReleaseAccess(data); } // ピクセルデータを更新する imageBitmap.WritePixels(imageRect, imageBuffer, DEPTH_WIDTH * BYTE_PER_PIXEL, 0); }
public void HandPipeLine() { PXCMSenseManager pp = m_form.Session.CreateSenseManager(); if (pp == null) { throw new Exception("PXCMSenseManager null"); } pp.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 360); //手 初始化 PXCMHandModule handAnalysis; PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler(); handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame); PXCMHandConfiguration handConfiguration = null; PXCMHandData handData = null; pxcmStatus status = pp.EnableHand(); handAnalysis = pp.QueryHand(); if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null) { Console.WriteLine("hand module load failed"); return; } handConfiguration = handAnalysis.CreateActiveConfiguration(); if (handConfiguration == null) { Console.WriteLine("Failed Create Configuration"); return; } handData = handAnalysis.CreateOutput(); if (handData == null) { Console.WriteLine("Failed Create Output"); return; } if (pp.Init(handler) != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("init failed"); return; } if (handConfiguration != null) { PXCMHandData.TrackingModeType trackingMode = PXCMHandData.TrackingModeType.TRACKING_MODE_FULL_HAND; // 配置收的Tracking Mode trackingMode = PXCMHandData.TrackingModeType.TRACKING_MODE_FULL_HAND; handConfiguration.SetTrackingMode(trackingMode); handConfiguration.EnableAllAlerts(); handConfiguration.EnableSegmentationImage(true); bool isEnabled = handConfiguration.IsSegmentationImageEnabled(); handConfiguration.ApplyChanges(); int totalNumOfGestures = handConfiguration.QueryGesturesTotalNumber(); if (totalNumOfGestures > 0) { for (int i = 0; i < totalNumOfGestures; i++) { string gestureName = string.Empty; if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) == pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine(gestureName); } } } } int frameCounter = 0; int frameNumber = 0; while (!m_form.Stopped) { string gestureName = "fist"; if (handConfiguration != null) { if (string.IsNullOrEmpty(gestureName) == false) { if (handConfiguration.IsGestureEnabled(gestureName) == false) { handConfiguration.DisableAllGestures(); handConfiguration.EnableGesture(gestureName, true); handConfiguration.ApplyChanges(); } } else { handConfiguration.DisableAllGestures(); handConfiguration.ApplyChanges(); } } if (pp.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR) { break; } frameCounter++; if (pp.IsConnected()) { PXCMCapture.Sample sample; sample = pp.QueryHandSample(); if (sample != null && sample.depth != null) { // frameNumber = liveCamera ? frameCounter : instance.captureManager.QueryFrameIndex(); frameNumber = frameCounter; } //bool b=(sample.ir==null); //b = (sample.color== null); //b = (sample.left == null); //b = (sample.right == null); DisplayPicture(sample.depth); if (handData != null) { handData.Update(); SaveHandData(handData); } m_form.UpdatePic(); } pp.ReleaseFrame(); } if (handData != null) { handData.Dispose(); } if (handConfiguration != null) { handConfiguration.Dispose(); } pp.Close(); pp.Dispose(); }
private void ProcessingHandThread() { // Start AcquireFrame/ReleaseFrame loop while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { hand = senseManager.QueryHand(); if (hand != null) { // Retrieve the most recent processed data handData = hand.CreateOutput(); handData.Update(); // Get number of tracked hands nhands = handData.QueryNumberOfHands(); if (nhands > 0) { // Retrieve hand identifier handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId); // Retrieve hand data handData.QueryHandDataById(handId, out ihand); // Retrieve all hand joint data for (int i = 0; i < nhands; i++) { for (int j = 0; j < 0x20; j++) { PXCMHandData.JointData jointData; ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData); nodes[i][j] = jointData; } } // Get world coordinates for tip of middle finger on the first hand in camera range handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x; handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y; handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z; // Retrieve gesture data if (handData.IsGestureFired("spreadfingers", out gestureData)) { gesture = Gesture.FingerSpread; } else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData)) { gesture = Gesture.Pinch; } else if (handData.IsGestureFired("wave", out gestureData)) { gesture = Gesture.Wave; } } else { gesture = Gesture.Undefined; } // Get alert status for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++) { PXCMHandData.AlertData alertData; if (handData.QueryFiredAlertData(i, out alertData) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } //Displaying last alert switch (alertData.label) { case PXCMHandData.AlertType.ALERT_HAND_DETECTED: detectionAlert = "Hand Detected"; detectionStatusOk = true; break; case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED: detectionAlert = "Hand Not Detected"; detectionStatusOk = false; break; case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED: calibrationAlert = "Hand Calibrated"; calibrationStatusOk = true; break; case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED: calibrationAlert = "Hand Not Calibrated"; calibrationStatusOk = false; break; case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS: bordersAlert = "Hand Inside Borders"; borderStatusOk = true; break; case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS: bordersAlert = "Hand Out Of Borders"; borderStatusOk = false; break; } } UpdateUI(); if (handData != null) handData.Dispose(); } senseManager.ReleaseFrame(); } }
public void ProcessingThread() { // Start AcquireFrame/ReleaseFrame loop while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { hand = senseManager.QueryHand(); if (hand != null) { // Retrieve the most recent processed data handData = hand.CreateOutput(); handData.Update(); // Get number of tracked hands nhands = handData.QueryNumberOfHands(); if (nhands > 0) { // Retrieve hand identifier handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId); // Retrieve hand data handData.QueryHandDataById(handId, out ihand); PXCMHandData.BodySideType bodySideType = ihand.QueryBodySide(); if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { leftHand = true; } else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { leftHand = false; } // Retrieve all hand joint data for (int i = 0; i < nhands; i++) { for (int j = 0; j < 0x20; j++) { PXCMHandData.JointData jointData; ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData); nodes[i][j] = jointData; } } // Get world coordinates for tip of middle finger on the first hand in camera range handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x; handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y; handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z; swipehandTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.x; swipehandTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.y; swipehandTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.z; //Console.Out.WriteLine("Before x={0}", swipehandTipX); //Console.Out.WriteLine("Before speed={0}", nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].speed.x); // Retrieve gesture data if (handData.IsGestureFired("spreadfingers", out gestureData)) { gesture = Gesture.FingerSpread; } else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData)) { gesture = Gesture.Pinch; } else if (handData.IsGestureFired("wave", out gestureData)) { gesture = Gesture.Wave; } else if (handData.IsGestureFired("swipe_left", out gestureData)) { gesture = Gesture.SwipeLeft; } else if (handData.IsGestureFired("swipe_right", out gestureData)) { gesture = Gesture.SwipeRight; } else if (handData.IsGestureFired("fist", out gestureData)) { gesture = Gesture.Fist; } else if (handData.IsGestureFired("thumb_up", out gestureData)) { gesture = Gesture.Thumb; } } else { gesture = Gesture.Undefined; } UpdateUI(); if (handData != null) { handData.Dispose(); } } senseManager.ReleaseFrame(); } }
/* Using PXCMSenseManager to handle data */ public void SimplePipeline() { form.UpdateInfo(String.Empty, Color.Black); bool liveCamera = false; bool flag = true; PXCMSenseManager instance = null; _disconnected = false; instance = form.g_session.CreateSenseManager(); if (instance == null) { form.UpdateStatus("Failed creating SenseManager"); return; } if (form.GetRecordState()) { instance.captureManager.SetFileName(form.GetFileName(), true); PXCMCapture.DeviceInfo info; if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info)) { instance.captureManager.FilterByDeviceInfo(info); } } else if (form.GetPlaybackState()) { instance.captureManager.SetFileName(form.GetFileName(), false); instance.captureManager.SetRealtime(false); } else { PXCMCapture.DeviceInfo info; if (String.IsNullOrEmpty(form.GetCheckedDevice())) { form.UpdateStatus("Device Failure"); return; } if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info)) { instance.captureManager.FilterByDeviceInfo(info); } liveCamera = true; } /* Set Module */ pxcmStatus status = instance.EnableHand(form.GetCheckedModule()); PXCMHandModule handAnalysis = instance.QueryHand(); if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null) { form.UpdateStatus("Failed Loading Module"); return; } PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler(); handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame); PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration(); PXCMHandData handData = handAnalysis.CreateOutput(); if (handConfiguration == null) { form.UpdateStatus("Failed Create Configuration"); return; } if (handData == null) { form.UpdateStatus("Failed Create Output"); return; } if (form.getInitGesturesFirstTime() == false) { int totalNumOfGestures = handConfiguration.QueryGesturesTotalNumber(); if (totalNumOfGestures > 0) { this.form.UpdateGesturesToList("", 0); for (int i = 0; i < totalNumOfGestures; i++) { string gestureName = string.Empty; if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) == pxcmStatus.PXCM_STATUS_NO_ERROR) { this.form.UpdateGesturesToList(gestureName, i + 1); } } form.setInitGesturesFirstTime(true); form.UpdateGesturesListSize(); } } FPSTimer timer = new FPSTimer(form); form.UpdateStatus("Init Started"); if (handAnalysis != null && instance.Init(handler) == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMCapture.DeviceInfo dinfo; PXCMCapture.Device device = instance.captureManager.device; if (device != null) { pxcmStatus result = device.QueryDeviceInfo(out dinfo); if (result == pxcmStatus.PXCM_STATUS_NO_ERROR && dinfo != null && dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM) { device.SetDepthConfidenceThreshold(1); device.SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED); device.SetIVCAMFilterOption(6); } _maxRange = device.QueryDepthSensorRange().max; } if (handConfiguration != null) { handConfiguration.EnableAllAlerts(); handConfiguration.EnableSegmentationImage(true); handConfiguration.ApplyChanges(); handConfiguration.Update(); } form.UpdateStatus("Streaming"); int frameCounter = 0; int frameNumber = 0; string nextPageGesture, previousPageGesture, firstPageGesture, endPageGesture; HandsRecognition.Hand nextHand, previousHand, firstHand, endHand; while (!form.stop) { form.GetHandType(out nextHand, out previousHand, out firstHand, out endHand); form.GetGestureName(out nextPageGesture, out previousPageGesture, out firstPageGesture, out endPageGesture); handConfiguration.DisableAllGestures(); if (string.IsNullOrEmpty(nextPageGesture) == false && handConfiguration.IsGestureEnabled(nextPageGesture) == false) { handConfiguration.EnableGesture(nextPageGesture, true); NextPageGesture.Gesture = nextPageGesture; NextPageGesture.handler = form.NextPage; NextPageGesture.HandType = nextHand; } if (string.IsNullOrEmpty(previousPageGesture) == false && handConfiguration.IsGestureEnabled(previousPageGesture) == false) { handConfiguration.EnableGesture(previousPageGesture, true); PreviousPageGesture.Gesture = previousPageGesture; PreviousPageGesture.handler = form.PreviousPage; PreviousPageGesture.HandType = previousHand; } if (string.IsNullOrEmpty(firstPageGesture) == false && handConfiguration.IsGestureEnabled(firstPageGesture) == false) { handConfiguration.EnableGesture(firstPageGesture, true); FirstPageGesture.Gesture = firstPageGesture; FirstPageGesture.handler = form.FirstPage; FirstPageGesture.HandType = firstHand; } if (string.IsNullOrEmpty(endPageGesture) == false && handConfiguration.IsGestureEnabled(endPageGesture) == false) { handConfiguration.EnableGesture(endPageGesture, true); EndPageGesture.Gesture = endPageGesture; EndPageGesture.handler = form.EndPage; EndPageGesture.HandType = endHand; } handConfiguration.ApplyChanges(); if (instance.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR) { break; } frameCounter++; if (!DisplayDeviceConnection(!instance.IsConnected())) { if (handData != null) { handData.Update(); } PXCMCapture.Sample sample = instance.QueryHandSample(); if (sample != null && sample.depth != null) { DisplayPicture(sample.depth, handData); if (handData != null) { frameNumber = liveCamera ? frameCounter : instance.captureManager.QueryFrameIndex(); DisplayJoints(handData); DisplayGesture(handData, frameNumber); DisplayAlerts(handData, frameNumber); } form.UpdatePanel(); } timer.Tick(); } instance.ReleaseFrame(); } // Clean Up if (handData != null) { handData.Dispose(); } if (handConfiguration != null) { handConfiguration.Dispose(); } } else { form.UpdateStatus("Init Failed"); flag = false; } foreach (PXCMImage pxcmImage in m_images) { pxcmImage.Dispose(); } instance.Close(); instance.Dispose(); if (flag) { form.UpdateStatus("Stopped"); } }
public Main.HandCoord[] GetJointCoordinates() { #region Catch exception errors /* Wait until frame is available and SenseManager is ready */ /* Make sure SenseManager has an instance */ if (SenseManager == null) { return(null); } /* Wait until any frame data is available */ if (SenseManager.AcquireFrame(true) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return(null); } #endregion #region Get joint coordinates HandModule = SenseManager.QueryHand(); // Get hand tracking Module instance HandData = HandModule.CreateOutput(); // Get hand tracking data if (HandData != null) { HandData.Update(); // Update hand data to the most current output /*Reset the detected hands to zero*/ for (int i = 0; i < NumOfHands; i++) { Coordinates[i].HandDetected = false; } /* Retrieve all joint data */ for (int i = 0; i < HandData.QueryNumberOfHands(); i++) { Coordinates[i].HandDetected = true; PXCMHandData.IHand iHand; Status = HandData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out iHand); if (Status != pxcmStatus.PXCM_STATUS_NO_ERROR) { Debug.LogError(Status); } else { for (int j = 0; j < NumOfJoints; j++) { Status = iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out JointData[i][j]); if (Status == pxcmStatus.PXCM_STATUS_NO_ERROR) { Coordinates[i].Coordinates[0, j] = JointData[i][j].positionWorld.x * -100; Coordinates[i].Coordinates[1, j] = JointData[i][j].positionWorld.y * 100; Coordinates[i].Coordinates[2, j] = JointData[i][j].positionWorld.z * 100; Coordinates[i].Coordinates[3, j] = JointData[i][j].speed.x * -100; Coordinates[i].Coordinates[4, j] = JointData[i][j].speed.y * -100; Coordinates[i].Coordinates[5, j] = JointData[i][j].speed.z * 100; } } } } } SenseManager.ReleaseFrame(); // Prepare for a new frame return(Coordinates); #endregion }
void Update() { //Dynamically Pause/Enable Modules int numberOfEnabledModules = 0; foreach (var option in _senseOptions) { if (option.RefCounter == 0 && option.Enabled) { if (option.ModuleCUID > 0) { SenseManager.PauseModule(option.ModuleCUID, true); } option.Enabled = false; } else if (option.RefCounter > 0 && !option.Enabled) { if (!option.Initialized) { OnDisable(); OnEnable(); Start(); } if (option.ModuleCUID > 0) { SenseManager.PauseModule(option.ModuleCUID, false); } option.Enabled = true; } if (option.Enabled) { numberOfEnabledModules++; } } //Update Speech commands if changed if (_speechCommandsChanged) { UpdateSpeechCommands(); SpeechManager.Reset(); } // Every frame update all the data if (Initialized && numberOfEnabledModules > 0) { _sts = SenseManager.AcquireFrame(true, 100); if (_sts == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled) { if (ImageRgbOutput != null) { ImageRgbOutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.color != null) { ImageRgbOutput = _captureSample.color; ImageRgbOutput.QueryInstance <PXCMAddRef>().AddRef(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled) { if (ImageDepthOutput != null) { ImageDepthOutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.depth != null) { ImageDepthOutput = _captureSample.depth; ImageDepthOutput.QueryInstance <PXCMAddRef>().AddRef(); if (PointCloud == null) { PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled) { if (PointCloud == null) { PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } _sts = _projection.QueryVertices(ImageDepthOutput, PointCloud); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.UVMap).Enabled) { if (UvMap == null) { UvMap = new PXCMPointF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } _projection.QueryUVMap(ImageDepthOutput, UvMap); } } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled) { if (ImageIROutput != null) { ImageIROutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.ir != null) { ImageIROutput = _captureSample.ir; ImageIROutput.QueryInstance <PXCMAddRef>().AddRef(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled) { FaceModuleOutput.Update(); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled) { HandDataOutput.Update(); } _captureSample = null; SenseManager.ReleaseFrame(); } //Speech if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled) { SpeechManager.QueryRecognizedCommands(out SpeechOutput); } } }
pxcmStatus newHandFrame(PXCMHandModule hand) { if (hand != null) { PXCMHandData handData = hand.CreateOutput(); handData.Update(); PXCMHandData.IHand iHandDataLeft = null, iHandDataRight = null; PXCMHandData.JointData jointData = null; PXCMImage image = null; handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out iHandDataLeft); handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out iHandDataRight); if (handForm != null && !handForm.IsDisposed) { this.handForm.HandCount = handData.QueryNumberOfHands(); if (iHandDataLeft != null) { iHandDataLeft.QuerySegmentationImage(out image); if (image != null) { PXCMImage.ImageData data = new PXCMImage.ImageData(); image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data); handForm.LeftHand = data.ToBitmap(0, image.info.width, image.info.height); image.ReleaseAccess(data); } } if (iHandDataRight != null) { iHandDataRight.QuerySegmentationImage(out image); if (image != null) { PXCMImage.ImageData data = new PXCMImage.ImageData(); image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data); handForm.RightHand = data.ToBitmap(0, image.info.width, image.info.height); image.ReleaseAccess(data); } } } if (iHandDataLeft != null) { if (jointData == null) { iHandDataLeft.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData); } } if (iHandDataRight != null) { if (jointData == null) { iHandDataRight.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData); } } if (jointData != null && canTrack.Checked) { Cursor.Position = new System.Drawing.Point( (int)((640.0f - jointData.positionImage.x) * Screen.PrimaryScreen.Bounds.Width / 640.0f), (int)(jointData.positionImage.y * Screen.PrimaryScreen.Bounds.Height / 480.0f)); PXCMHandData.GestureData gestureData = null; if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData)) { Program.DoMouseClick(); } Console.WriteLine("Z Position: " + jointData.positionWorld.z); } handData.Dispose(); } return(pxcmStatus.PXCM_STATUS_NO_ERROR); }
// Update is called once per frame void Update() { /* Make sure SenseManager Instance is valid */ if (sm == null) { return; } /* Wait until any frame data is available */ if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } /* Retrieve hand tracking Module Instance */ handAnalyzer = sm.QueryHand(); if (handAnalyzer != null) { /* Retrieve hand tracking Data */ PXCMHandData _handData = handAnalyzer.CreateOutput(); if (_handData != null) { _handData.Update(); /* Retrieve all joint Data */ bool[] someJointsDetected = { false, false }; for (int i = 0; i < _handData.QueryNumberOfHands(); i++) { PXCMHandData.IHand _iHand; if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR) { for (int j = 0; j < MaxJoints; j++) { if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i, j]) != pxcmStatus.PXCM_STATUS_NO_ERROR) { jointData[i, j] = null; } else if (jointData[i, j].confidence == 100) { someJointsDetected[i] = true; } } if (!handList.ContainsKey(_iHand.QueryUniqueId())) { handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide()); } } } for (int i = 0; i < MaxHands; i++) { if (!someJointsDetected[i] && handWasTracked[i]) { for (int j = 0; j < MaxJoints; j++) { Joint joint = myJoints[i, j]; if (joint == null) { continue; } joint.transform.position = joint.originalPosition; joint.transform.rotation = joint.originalRotation; foreach (Joint e in joint.extensions) { e.transform.position = e.originalPosition; e.transform.rotation = e.originalRotation; } } } else { if (!handWasTracked[i]) { if (myJoints[i, 0] == null || jointData[i, 0] == null || jointData[i, 0].confidence < 100) { break; } PXCMPoint3DF32 smoothedPoint = smoother3D[i, 0].SmoothValue(jointData[i, 0].positionWorld); Vector3 targetPosition = new Vector3(-1 * smoothedPoint.x, smoothedPoint.y, smoothedPoint.z); trackingOffset[i] = targetPosition - myJoints[i, 0].originalPosition; } /* Smooth the data and move the joints*/ MoveJoints(i); } handWasTracked[i] = someJointsDetected[i]; } } handAnalyzer.Dispose(); } sm.ReleaseFrame(); RotateCam(); }
// Update is called once per frame void Update() { // For testing only. //handRenderer.queryLeftHand2DCoordinates (); _outputData = SenseToolkitManager.Instance.HandDataOutput; if (_outputData != null) { _outputData.Update(); /* Retrieve Hand Joints*/ joints = new PXCMHandData.JointData[2, PXCMHandData.NUMBER_OF_JOINTS]; for (int i = 0; i < _outputData.QueryNumberOfHands(); i++) { PXCMHandData.IHand _handData; _outputData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _handData); //Debug.Log(_handData.IsCalibrated().ToString()); //if(_handData!=null){ //else { Debug.Log ("No data"); continue; } for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++) { _handData.QueryTrackedJoint((PXCMHandData.JointType)j, out joints[i, j]); } handIds[i] = _handData.QueryUniqueId(); bodySides[i] = _handData.QueryBodySide(); if (!_handData.IsCalibrated()) { if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { isleftCalibrated = false; } else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { isrightCalibrated = false; } } else { if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { isleftCalibrated = true; } else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { isrightCalibrated = true; } } } handRenderer.DisplaySmoothenedJoints(_outputData, joints, handIds, bodySides, _outputData.QueryNumberOfHands()); /* Retrieve Alert Data */ /* * PXCMHandData.AlertData _alertData; * for (int i = 0; i < _outputData.QueryFiredAlertsNumber(); i++) * if (_outputData.QueryFiredAlertData(i, out _alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR) * handRenderer.DisplayAlerts(_alertData); */ /* Retrieve Gesture Data */ PXCMHandData.GestureData _gestureData; for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++) { if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { handRenderer.DisplayGestures(_gestureData); } } if (_outputData.QueryNumberOfHands() == 0) { handRenderer.makeNull(); } } else { handRenderer.makeNull(); } // For testing //handRenderer.DisplayGest (); displayHandGestures(); }
private void ProcessingThread() { // Start AcquireFrame/ReleaseFrame loop - MAIN PROCESSING LOOP while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { if (firstFrame == true) { firstFrame = false; //pipeClient.SendMessage(CAMERA_CONNECTED_MESSAGE); } //Get sample from the sensemanager to convert to bitmap and show PXCMCapture.Sample sample = senseManager.QuerySample(); Bitmap colorBitmap; PXCMImage.ImageData colorData = null; // Get color/ir image data if (cameraMode == "Color") sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData); else if (cameraMode == "IR") sample.ir.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData); else if (cameraMode == "Depth") ;// -> broken! // sample.depth.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH, out colorData); else sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData); //convert it to bitmap colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); // Retrieve hand and face data AND EMOTION DATA hand = senseManager.QueryHand(); face = senseManager.QueryFace(); emotion = senseManager.QueryEmotion(); //Process hand data if (hand != null) { // Retrieve the most recent processed data handData = hand.CreateOutput(); handData.Update(); handWaving = handData.IsGestureFired("wave", out gestureData); } //Process face data if (face != null) { // Retrieve the most recent processed data faceData = face.CreateOutput(); faceData.Update(); numFacesDetected = faceData.QueryNumberOfDetectedFaces(); if (numFacesDetected > 0) { // for (Int32 i = 0; i < numFacesDetected; i++) --> MULTIPLE FACE DETECTION DISABLED, UNCOMMENT TO INCLUDE // { // PXCMFaceData.Face singleFace = faceData.QueryFaceByIndex(i); --> FOR MULTIPLE FACE DETECTION //get all possible data from frame PXCMFaceData.Face singleFaceData = faceData.QueryFaceByIndex(0); //only getting first face! PXCMFaceData.ExpressionsData singleExprData = singleFaceData.QueryExpressions(); PXCMFaceData.DetectionData detectionData = singleFaceData.QueryDetection(); PXCMFaceData.LandmarksData landmarksData = singleFaceData.QueryLandmarks(); PXCMFaceData.PoseData poseData = singleFaceData.QueryPose(); //Work on face location data from detectionData if (detectionData != null) { // vars are defined globally detectionData.QueryBoundingRect(out boundingRect); detectionData.QueryFaceAverageDepth(out averageDepth); } //Work on getting landmark data if (landmarksData != null) { //var is defined globally landmarksData.QueryPoints(out landmarkPoints); } //Work on getting euler angles for face pose data if (poseData != null) { //var is defined globally poseData.QueryPoseAngles(out eulerAngles); poseData.QueryPoseQuaternion(out quaternionAngles); } //Do work on all face location data from singleExprData if (singleExprData != null) { //get scores and intensities for right and left eye closing - 22 possible expressions --> put into hashtable PXCMFaceData.ExpressionsData.FaceExpressionResult score; //this gets a list of enum names as strings var enumNames = Enum.GetNames(typeof(PXCMFaceData.ExpressionsData.FaceExpression)); //for all enumnames, calculate the for (int j = 0; j < enumNames.Length; j++) { PXCMFaceData.ExpressionsData.FaceExpressionResult innerScore; singleExprData.QueryExpression((PXCMFaceData.ExpressionsData.FaceExpression)(j), out innerScore); //Console.WriteLine((PXCMFaceData.ExpressionsData.FaceExpression)(j)); exprTable[enumNames[j]] = innerScore.intensity; } //Attempt to write to file if there are any significant events /* //check if everything is 0 bool significantEntry = false; foreach (DictionaryEntry entry in exprTable) { if (Convert.ToInt32(entry.Value.ToString()) != 0) { significantEntry = true; break; } } if (significantEntry) */ writeSignificantToFile(exprTable, boundingRect, averageDepth, landmarkPoints, eulerAngles, quaternionAngles); singleExprData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT, out score); lEyeClosedIntensity = score.intensity; singleExprData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT, out score); rEyeClosedIntensity = score.intensity; //eye closed logic -> will be reset in UI thread after some number of frames if (lEyeClosedIntensity >= EYE_CLOSED_DETECT_THRESHOLD) lEyeClosed = true; if (rEyeClosedIntensity >= EYE_CLOSED_DETECT_THRESHOLD) rEyeClosed = true; } // } } } if (emotion != null) { int numFaces = emotion.QueryNumFaces(); for (int fid = 0; fid < numFaces; fid++) { //TODO - MULTIPLE FACE IMPLEMENTATION? //retrieve all est data PXCMEmotion.EmotionData[] arrData = new PXCMEmotion.EmotionData[10]; emotion.QueryAllEmotionData(fid, out arrData); //find emotion with maximum evidence int idx_outstanding_emotion = 0; int max_evidence = arrData[0].evidence; for (int k = 1; k < 7; k++) { if (arrData[k].evidence < max_evidence) { } else { max_evidence = arrData[k].evidence; idx_outstanding_emotion = k; } } currentEmotion = arrData[idx_outstanding_emotion].eid; //Console.WriteLine(currentEmotion.ToString()); emotionEvidence = max_evidence; // Console.WriteLine(currentEmotion.ToString() + ":" + emotionEvidence.ToString()); } } // Update the user interface UpdateUI(colorBitmap); // Release the frame if (handData != null) handData.Dispose(); // colorBitmap.Dispose(); sample.color.ReleaseAccess(colorData); senseManager.ReleaseFrame(); } }