public string findVovel(PXCMHandData.JointData[][] nodes, int tip) { PXCMPoint3DF32 tipPoint = nodes[1][5 + 4 * tip].positionWorld; float minTipDistance = 1000; int tipIndex = -1; for (int i = 0; i < 5; i++) { float tipDistance = Distance(tipPoint, nodes[0][5 + i * 4].positionWorld); if (tipDistance < minTipDistance) { minTipDistance = tipDistance; tipIndex = i; } } switch (tipIndex) { case 0: return "A"; case 1: return "E"; case 2: return "I"; case 3: return "O"; case 4: return "U"; default: return "?"; } }
protected override bool isBad(PXCMHandData.JointData[] data){ Double DeltaIndex = Math.Sqrt(Math.Pow(data [8].positionWorld.x - data [6].positionWorld.x,2) +Math.Pow(data [8].positionWorld.z - data [6].positionWorld.z,2)); // myTextLeft.text = DeltaIndex.ToString (); if (DeltaIndex > MaxDelta) { return true; } Double DeltaRing = Math.Sqrt(Math.Pow(data [16].positionWorld.x - data [14].positionWorld.x,2) +Math.Pow(data [16].positionWorld.z - data [14].positionWorld.z,2)); // myTextLeft.text += "\n"+DeltaRing.ToString (); if (DeltaRing > MaxDelta) { return true; } Double DeltaPinky = Math.Sqrt(Math.Pow(data [20].positionWorld.x - data [18].positionWorld.x,2) +Math.Pow(data [20].positionWorld.z - data [18].positionWorld.z,2)); // myTextLeft.text += "\n"+DeltaRing.ToString (); if (DeltaPinky > MaxDelta) { return true; } Double angle1 = 2 * Math.Acos(data[11].localRotation.w); Double angle2 = 2 * Math.Acos (data [12].localRotation.w); if (angle1 >= 0.5 || (angle2 <= 0.5 && angle2 >= 2)) return true; return false; }
public static int JointToInt(PXCMHandData.JointType label) { int jointLabel = -1; switch (label) { case PXCMHandData.JointType.JOINT_THUMB_TIP: { jointLabel = 0; break; } case PXCMHandData.JointType.JOINT_INDEX_BASE: { jointLabel = 1; break; } case PXCMHandData.JointType.JOINT_MIDDLE_TIP: { jointLabel = 2; break; } case PXCMHandData.JointType.JOINT_RING_TIP: { jointLabel = 3; break; } case PXCMHandData.JointType.JOINT_PINKY_TIP: { jointLabel = 4; break; } case PXCMHandData.JointType.JOINT_WRIST: case PXCMHandData.JointType.JOINT_CENTER: case PXCMHandData.JointType.JOINT_THUMB_BASE: case PXCMHandData.JointType.JOINT_THUMB_JT1: case PXCMHandData.JointType.JOINT_THUMB_JT2: case PXCMHandData.JointType.JOINT_INDEX_TIP: case PXCMHandData.JointType.JOINT_INDEX_JT1: case PXCMHandData.JointType.JOINT_INDEX_JT2: case PXCMHandData.JointType.JOINT_MIDDLE_BASE: case PXCMHandData.JointType.JOINT_MIDDLE_JT1: case PXCMHandData.JointType.JOINT_MIDDLE_JT2: case PXCMHandData.JointType.JOINT_RING_BASE: case PXCMHandData.JointType.JOINT_RING_JT1: case PXCMHandData.JointType.JOINT_RING_JT2: case PXCMHandData.JointType.JOINT_PINKY_BASE: case PXCMHandData.JointType.JOINT_PINKY_JT1: case PXCMHandData.JointType.JOINT_PINKY_JT2: { jointLabel = -1; break; } } return jointLabel; }
private void TrackIndex(Finger finger, PXCMHandData.IHand handInfo) { SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_BASE, finger.BaseJoint); SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_JT1, finger.FirstJoint); SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_JT2, finger.SecondJoint); SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_TIP, finger); SetFingerOpenness(finger, PXCMHandData.FingerType.FINGER_INDEX, handInfo); }
protected override void getStatus(PXCMHandData.JointData[] data, bool isLeft){ if (isLeft) lhstatus = checkMotion (data); if (lhstatus != Status.None && lhstatus > BestPerform) BestPerform = lhstatus; label1.GetComponent<UILabel>().text = lhstatus.ToString(); }
private void SetHandPosition(Hand hand, PXCMHandData.IHand handInfo) { var world = handInfo.QueryMassCenterWorld(); if (_smoothers.ContainsKey(hand)) { world = _smoothers[hand].SmoothValue(world); } var imagePosition = ToPoint3D(handInfo.QueryMassCenterImage()); var worldPosition = ToPoint3D(world); hand.Position = CreatePosition(imagePosition, worldPosition); }
protected override Status checkMotion(PXCMHandData.JointData[] data){ if (isBad (data)) return Status.Bad; if (isGreat (data)) return Status.Great; if (isGood (data)) return Status.Good; return Status.None; }
private void OnFiredAlert(PXCMHandData.AlertData alertData) { switch (alertData.label) { case PXCMHandData.AlertType.ALERT_HAND_DETECTED: calibrated = true; break; case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_TRACKED: calibrated = true; break; case PXCMHandData.AlertType.ALERT_HAND_NOT_TRACKED: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED: calibrated = true; break; case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS: calibrated = true; break; case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_LEFT_BORDER: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_RIGHT_BORDER: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_TOP_BORDER: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BOTTOM_BORDER: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_TOO_FAR: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_TOO_CLOSE: calibrated = false; break; case PXCMHandData.AlertType.ALERT_HAND_LOW_CONFIDENCE: calibrated = false; break; default: calibrated = false; throw new ArgumentOutOfRangeException(); } }
private void OnGesture(PXCMHandData.GestureData gesturedata) { string g = $"Gesture: {gesturedata.name}-{gesturedata.handId}-{gesturedata.state}"; Debug.WriteLine(g); switch (gesturedata.name) { case "wave": //_gestures.OnWave(new GestureEventArgs("wave")); return; //case "swipe_left": // _sensor.OnSlideLeft(new GestureEventArgs("Swipe Left")); // return; } }
private void ExtractHand(PXCMHandData handData, Hand hand, PXCMHandData.AccessOrderType access) { PXCMHandData.IHand handInfo; if (handData.QueryHandData(access, 0, out handInfo) != NoError) { hand.Visible = false; return; } hand.Visible = true; var openness = handInfo.QueryOpenness(); hand.Open = openness > 75; var position = handInfo.QueryMassCenterWorld(); hand.Distance = (int)(position.z * 100); }
private int weightsNum = 4; //smoothing factor #endregion Fields #region Methods //Display Gestures void DisplayGestures(PXCMHandData.GestureData gestureData) { //if (handList.ContainsKey (gestureData.handId)) { //switch ((PXCMHandData.BodySideType)handList [gestureData.handId]) { //case PXCMHandData.BodySideType.BODY_SIDE_LEFT: // myTextLeft.text = gestureData.name.ToString (); // break; //case PXCMHandData.BodySideType.BODY_SIDE_RIGHT: // myTextRight.text = gestureData.name.ToString (); // break; //} //} }
private void SetHandOrientation(Hand hand, PXCMHandData.IHand handInfo) { var d4 = handInfo.QueryPalmOrientation(); PXCMRotation rotationHelper; _camera.Session.CreateImpl(out rotationHelper); rotationHelper.SetFromQuaternion(d4); var rotationEuler = rotationHelper.QueryEulerAngles(PXCMRotation.EulerOrder.PITCH_YAW_ROLL); var x = rotationEuler.x * 180 / Math.PI; var y = rotationEuler.y * 180 / Math.PI; var z = rotationEuler.z * 180 / Math.PI; hand.Rotation = new Rotation(x, y, z); rotationHelper.Dispose(); }
private void onFiredGesture(PXCMHandData.GestureData gestureData) { if (gestureData.name.CompareTo("spreadfingers") == 0 && this.isPlaying) { this.BackColor = Color.Red; this.TogglePlay(); } if (gestureData.name.CompareTo("thumb_up") == 0 && !this.isPlaying) { this.BackColor = Color.Green; this.TogglePlay(); } }
private void TrackHandAndFingers(Hand hand, PXCMHandData data, PXCMHandData.AccessOrderType label) { PXCMHandData.IHand handInfo; if (data.QueryHandData(label, 0, out handInfo) != NoError) { hand.IsVisible = false; return; } hand.IsVisible = true; SetHandOrientation(hand, handInfo); SetHandOpenness(hand, handInfo); SetHandPosition(hand, handInfo); TrackIndex(hand.Index, handInfo); TrackMiddle(hand.Middle, handInfo); TrackRing(hand.Ring, handInfo); TrackPinky(hand.Pinky, handInfo); TrackThumb(hand.Thumb, handInfo); }
public Camera(params PXCMHandConfiguration.OnFiredGestureDelegate[] dlgts) { // Create the manager this._session = PXCMSession.CreateInstance(); this._mngr = this._session.CreateSenseManager(); // streammmm PXCMVideoModule.DataDesc desc = new PXCMVideoModule.DataDesc(); desc.deviceInfo.streams = PXCMCapture.StreamType.STREAM_TYPE_COLOR | PXCMCapture.StreamType.STREAM_TYPE_DEPTH; this._mngr.EnableStreams(desc); //this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, Camera.WIDTH, Camera.HEIGHT, 30); //this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, Camera.WIDTH, Camera.HEIGHT, 30); // Hands this._mngr.EnableHand(); this._hand = this._mngr.QueryHand(); this._handData = this._hand.CreateOutput(); // Hands config PXCMHandConfiguration conf = this._hand.CreateActiveConfiguration(); conf.EnableGesture("spreadfingers", false); conf.EnableGesture("thumb_up", false); // Subscribe hands alerts conf.EnableAllAlerts(); conf.SubscribeAlert(this.onFiredAlert); // Subscribe all gestures foreach (PXCMHandConfiguration.OnFiredGestureDelegate subscriber in dlgts) { conf.SubscribeGesture(subscriber); } // and the private one for debug conf.SubscribeGesture(this.onFiredGesture); // Apply it all conf.ApplyChanges(); // Set events this._handler = new PXCMSenseManager.Handler(); this._handler.onModuleProcessedFrame = this.onModuleProcessedFrame; this._mngr.Init(this._handler); }
public override void Listen() { // attach the controller to the PXCM sensor _senseManager = Session.CreateSenseManager(); _senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 30); _senseManager.EnableHand(); _handModule = _senseManager.QueryHand(); _handData = _handModule.CreateOutput(); _handConfiguration = _handModule.CreateActiveConfiguration(); _handConfiguration.SubscribeGesture(_handGestureHandler); _handConfiguration.SubscribeAlert(_handAlertHandler); _handConfiguration.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_TRACKED); _handConfiguration.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_CALIBRATED); _handConfiguration.EnableGesture("full_pinch"); _handConfiguration.EnableGesture("thumb_up"); _handConfiguration.ApplyChanges(); _senseManager.Init(_handler); sensorActive = true; _senseManager.StreamFrames(true); _senseManager.Close(); }
/* Displaying current frame gestures */ private void DisplayGesture(PXCMHandData handAnalysis,int frameNumber) { int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber(); string gestureStatusLeft = string.Empty; string gestureStatusRight = string.Empty; if (firedGesturesNumber == 0) { return; } for (int i = 0; i < firedGesturesNumber; i++) { PXCMHandData.GestureData gestureData; if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMHandData.IHand handData; if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR) return; PXCMHandData.BodySideType bodySideType = handData.QueryBodySide(); if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT) { gestureStatusLeft += "Left Hand Gesture: " + gestureData.name; } else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT) { gestureStatusRight += "Right Hand Gesture: " + gestureData.name; } } } if (gestureStatusLeft == String.Empty) form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusRight + "\n", Color.SeaGreen); else form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusLeft + ", " + gestureStatusRight + "\n", Color.SeaGreen); }
public static string AlertToString(PXCMHandData.AlertType label) { string alertLabel = ""; switch (label) { case PXCMHandData.AlertType.ALERT_HAND_DETECTED: { alertLabel = "ALERT_HAND_DETECTED"; break; } case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED: { alertLabel = "ALERT_HAND_NOT_DETECTED"; break; } case PXCMHandData.AlertType.ALERT_HAND_TRACKED: { alertLabel = "ALERT_HAND_TRACKED"; break; } case PXCMHandData.AlertType.ALERT_HAND_NOT_TRACKED: { alertLabel = "ALERT_HAND_NOT_TRACKED"; break; } case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED: { alertLabel = "ALERT_HAND_CALIBRATED"; break; } case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED: { alertLabel = "ALERT_HAND_NOT_CALIBRATED"; break; } case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS: { alertLabel = "ALERT_HAND_OUT_OF_BORDERS"; break; } case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS: { alertLabel = "ALERT_HAND_INSIDE_BORDERS"; break; } case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_LEFT_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_LEFT_BORDER"; break; } case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_RIGHT_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_RIGHT_BORDER"; break; } case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_TOP_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_TOP_BORDER"; break; } case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BOTTOM_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_BOTTOM_BORDER"; break; } case PXCMHandData.AlertType.ALERT_HAND_TOO_FAR: { alertLabel = "ALERT_HAND_TOO_FAR"; break; } case PXCMHandData.AlertType.ALERT_HAND_TOO_CLOSE: { alertLabel = "ALERT_HAND_TOO_CLOSE"; break; } case PXCMHandData.AlertType.ALERT_HAND_LOW_CONFIDENCE: { alertLabel = "ALERT_HAND_LOW_CONFIDENCE"; break; } } return alertLabel; }
private void OnFiredGesture(PXCMHandData.GestureData gestureData) { switch (gestureData.state) { case PXCMHandData.GestureStateType.GESTURE_STATE_START: break; case PXCMHandData.GestureStateType.GESTURE_STATE_IN_PROGRESS: break; case PXCMHandData.GestureStateType.GESTURE_STATE_END: break; default: throw new ArgumentOutOfRangeException(); } if (gestureData.name == "full_pinch") { _movementController.ActivateGripper(); } if (gestureData.name == "thumb_up") { _movementController.BlinkLight(); } }
public HandsRecognition(int mod) { try { session = PXCMSession.CreateInstance(); mode = mod; if (mod == 0) { rightcp = new checkPiano(); leftcp = new checkPiano(); } else if (mod == 1) { rightcp = new checkDrum(); leftcp = new checkDrum(); } _disconnected = false; instance = session.CreateSenseManager(); if (instance == null) { MessageBox.Show("Failed creating SenseManager", "OnAlert"); return; } /* Set Module */ pxcmStatus status = instance.EnableHand();//form.GetCheckedModule()); handAnalysis = instance.QueryHand(); if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null) { MessageBox.Show("Failed Loading Module", "OnAlert"); return; } handler = new PXCMSenseManager.Handler(); handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame); handData = handAnalysis.CreateOutput(); } catch { MessageBox.Show("Init Failed."); Environment.Exit(0); } }
/* Displaying current frames hand joints */ private void DisplayJoints(PXCMHandData handOutput, long timeStamp = 0) { //Iterate hands PXCMHandData.JointData[][] nodes = new PXCMHandData.JointData[][] { new PXCMHandData.JointData[0x20], new PXCMHandData.JointData[0x20] }; int numOfHands = handOutput.QueryNumberOfHands(); for (int i = 0; i < numOfHands; i++) { //Get hand by time of appearence PXCMHandData.IHand handData; if (handOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, i, out handData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (handData != null) { Coordinate coor; string handSide = "Unknown Hand"; handSide = handData.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT ? "Left Hand" : "Right Hand"; switch (handSide) { case ("Left Hand"): { coor.handside = 0; Rnum++; break; } case ("Right Hand"): { coor.handside = 1; Lnum++; break; } default: { coor.handside = 2; break; } } //Iterate Joints for (int j = 0; j < 0x20; j++) { sr = File.CreateText(FILE_NAME); PXCMHandData.JointData jointData; handData.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData); nodes[i][j] = jointData; if (handData.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData) == pxcmStatus.PXCM_STATUS_NO_ERROR && Handsinfo.JointToInt((PXCMHandData.JointType)j) != -1) { coor.finger = Handsinfo.JointToInt((PXCMHandData.JointType)j); coor.x = jointData.positionWorld.x; coor.y = jointData.positionWorld.y; coor.z = jointData.positionWorld.z; //right if (coor.finger == 1 && coor.handside == 1) { this.RHandMove(coor.x, coor.y); sr.WriteLine(coor.x + " " + coor.y + " " + coor.z + "\n"); MYPoint p = new MYPoint(coor.x, coor.y, coor.z); int num = rightcp.checkNote(p); if (num != 0 && Rnum < 2 && Lnum < 2) { DateTime DateTime2 = DateTime.Now; diff = tim.ExecTimeDiff(starttmp, DateTime2); this.HandClick(num, (int)diff); } } //left if (coor.finger == 1 && coor.handside == 0) { this.LHandMove(coor.x, coor.y); sr.WriteLine(coor.x + " " + coor.y + " " + coor.z + "\n"); MYPoint p = new MYPoint(coor.x, coor.y, coor.z); int num = leftcp.checkNote(p); if (num != 0 && Rnum < 2 && Lnum < 2) { DateTime DateTime2 = DateTime.Now; diff = tim.ExecTimeDiff(starttmp, DateTime2); this.HandClick(num, (int)diff); } } } sr.Close(); } // end iterating over joints } } // end itrating over hands } Lnum = 0; Rnum = 0; }
private static void ProcessHands(PXCMHandData handData) { // Querying how many hands were detected int numberOfHands = handData.QueryNumberOfHands(); //Console.WriteLine("{0} hand(s) were detected.", numberOfHands); // Querying the information about detected hands for (int i = 0; i < numberOfHands; i++) { // Querying hand id int handId; pxcmStatus queryHandIdStatus = handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, i, out handId); if (queryHandIdStatus != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("Failed to query the hand Id."); continue; } //Console.WriteLine("Hand id: {0}", handId); // Querying the hand data PXCMHandData.IHand hand; pxcmStatus queryHandStatus = handData.QueryHandDataById(handId, out hand); if (queryHandStatus == pxcmStatus.PXCM_STATUS_NO_ERROR && hand != null) { // Querying the body side (Left/Right) PXCMHandData.BodySideType bodySide = hand.QueryBodySide(); //Console.WriteLine("Body Side: {0}", bodySide); // Querying the hand openness int handOpenness = hand.QueryOpenness(); //Console.WriteLine("Hand openness: {0}", handOpenness); SendMqttMessage(String.Format("red?{0}", (255 * handOpenness) / 100)); // Querying Hand 2D Position PXCMPointF32 massCenterImage = hand.QueryMassCenterImage(); //Console.WriteLine("Hand position on image: {0} | {1}", massCenterImage.x, massCenterImage.y); PXCMPoint4DF32 palmOrientation = hand.QueryPalmOrientation(); rotationHelper.SetFromQuaternion(palmOrientation); PXCMPoint3DF32 rotationEuler = rotationHelper.QueryEulerAngles(); double angleInDegrees = (180 * rotationEuler.y / Math.PI) + 180; Console.WriteLine("Angle in degrees: {0}", angleInDegrees); SendMqttMessage(String.Format("red?{0}", (int)angleInDegrees)); // Console.WriteLine("Rotation x:{0},y:{1},z:{2}", rotationEuler.x, rotationEuler.y, rotationEuler.z); // Querying Hand 3D Position PXCMPoint3DF32 massCenterWorld = hand.QueryMassCenterWorld(); //Console.WriteLine("Hand position on world: {0} | {1} | {2}", massCenterWorld.x, massCenterWorld.y, massCenterWorld.z); /* * // Querying Hand Joints * if (hand.HasTrackedJoints()) * { * foreach (PXCMHandData.JointType jointType in Enum.GetValues(typeof(PXCMHandData.JointType))) * { * PXCMHandData.JointData jointData; * pxcmStatus queryStatus = hand.QueryTrackedJoint(jointType, out jointData); * * if (queryStatus == pxcmStatus.PXCM_STATUS_NO_ERROR && jointData != null) * { * // Printing joint label and tracking confidence * Console.WriteLine("Joint {0} with confidence {1}", jointType, jointData.confidence); * * // Printing the 2D position (image) * Console.WriteLine("\t2D Position: {0} | {1}", jointData.positionImage.x, jointData.positionImage.y); * * // Printing the 3D position (depth) * Console.WriteLine("\t3D Position: {0} | {1} | {2}", jointData.positionWorld.x, jointData.positionWorld.y, jointData.positionWorld.z); * } * } * }*/ } //Console.WriteLine("----------"); //Console.Clear(); } }
protected override bool isGreat(PXCMHandData.JointData[] data){ Double angle2 = 2 * Math.Acos (data [12].localRotation.w); if (angle2 > 1.4 && angle2 < 1.6) return true; return false; }
// Update is called once per frame void Update() { /* Make sure SenseManager Instance is valid */ if (sm == null) { return; } /* Wait until any frame data is available */ if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } /* Retrieve hand tracking Module Instance */ handAnalyzer = sm.QueryHand(); if (handAnalyzer != null) { /* Retrieve hand tracking Data */ PXCMHandData _handData = handAnalyzer.CreateOutput(); if (_handData != null) { _handData.Update(); /* Retrieve all joint Data */ bool[] someJointsDetected = { false, false }; for (int i = 0; i < _handData.QueryNumberOfHands(); i++) { PXCMHandData.IHand _iHand; if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR) { for (int j = 0; j < MaxJoints; j++) { if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i, j]) != pxcmStatus.PXCM_STATUS_NO_ERROR) { jointData[i, j] = null; } else if (jointData[i, j].confidence == 100) { someJointsDetected[i] = true; } } if (!handList.ContainsKey(_iHand.QueryUniqueId())) { handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide()); } } } for (int i = 0; i < MaxHands; i++) { if (!someJointsDetected[i] && handWasTracked[i]) { for (int j = 0; j < MaxJoints; j++) { Joint joint = myJoints[i, j]; if (joint == null) { continue; } joint.transform.position = joint.originalPosition; joint.transform.rotation = joint.originalRotation; foreach (Joint e in joint.extensions) { e.transform.position = e.originalPosition; e.transform.rotation = e.originalRotation; } } } else { if (!handWasTracked[i]) { if (myJoints[i, 0] == null || jointData[i, 0] == null || jointData[i, 0].confidence < 100) { break; } PXCMPoint3DF32 smoothedPoint = smoother3D[i, 0].SmoothValue(jointData[i, 0].positionWorld); Vector3 targetPosition = new Vector3(-1 * smoothedPoint.x, smoothedPoint.y, smoothedPoint.z); trackingOffset[i] = targetPosition - myJoints[i, 0].originalPosition; } /* Smooth the data and move the joints*/ MoveJoints(i); } handWasTracked[i] = someJointsDetected[i]; } } handAnalyzer.Dispose(); } sm.ReleaseFrame(); RotateCam(); }
public void DisplayJoints(PXCMHandData.JointData[][] nodes, int numOfHands) { if (bitmap == null) return; if (nodes == null) return; if (Joints.Checked || Skeleton.Checked) { lock (this) { int scaleFactor = 1; Graphics g = Graphics.FromImage(bitmap); using (Pen boneColor = new Pen(Color.DodgerBlue, 3.0f)) { for (int i = 0; i < numOfHands; i++) { if (nodes[i][0] == null) continue; int baseX = (int) nodes[i][0].positionImage.x/scaleFactor; int baseY = (int) nodes[i][0].positionImage.y/scaleFactor; int wristX = (int) nodes[i][0].positionImage.x/scaleFactor; int wristY = (int) nodes[i][0].positionImage.y/scaleFactor; if (Skeleton.Checked) { for (int j = 1; j < 22; j++) { if (nodes[i][j] == null) continue; int x = (int) nodes[i][j].positionImage.x/scaleFactor; int y = (int) nodes[i][j].positionImage.y/scaleFactor; if (nodes[i][j].confidence <= 0) continue; if (j == 2 || j == 6 || j == 10 || j == 14 || j == 18) { baseX = wristX; baseY = wristY; } g.DrawLine(boneColor, new Point(baseX, baseY), new Point(x, y)); baseX = x; baseY = y; } } if (Joints.Checked) { using ( Pen red = new Pen(Color.Red, 3.0f), black = new Pen(Color.Black, 3.0f), green = new Pen(Color.Green, 3.0f), blue = new Pen(Color.Blue, 3.0f), cyan = new Pen(Color.Cyan, 3.0f), yellow = new Pen(Color.Yellow, 3.0f), orange = new Pen(Color.Orange, 3.0f)) { Pen currnetPen = black; for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++) { float sz = 4; if (Labelmap.Checked) sz = 2; int x = (int) nodes[i][j].positionImage.x/scaleFactor; int y = (int) nodes[i][j].positionImage.y/scaleFactor; if (nodes[i][j].confidence <= 0) continue; //Wrist if (j == 0) { currnetPen = black; } //Center if (j == 1) { currnetPen = red; sz += 4; } //Thumb if (j == 2 || j == 3 || j == 4 || j == 5) { currnetPen = green; } //Index Finger if (j == 6 || j == 7 || j == 8 || j == 9) { currnetPen = blue; } //Finger if (j == 10 || j == 11 || j == 12 || j == 13) { currnetPen = yellow; } //Ring Finger if (j == 14 || j == 15 || j == 16 || j == 17) { currnetPen = cyan; } //Pinkey if (j == 18 || j == 19 || j == 20 || j == 21) { currnetPen = orange; } if (j == 5 || j == 9 || j == 13 || j == 17 || j == 21) { sz += 4; //currnetPen.Width = 1; } g.DrawEllipse(currnetPen, x - sz/2, y - sz/2, sz, sz); } } } } } g.Dispose(); } } }
protected virtual bool isPrepared(PXCMHandData.JointData[] data){ return false; }
private void ProcessingHandThread() { // Start AcquireFrame/ReleaseFrame loop while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { hand = senseManager.QueryHand(); if (hand != null) { // Retrieve the most recent processed data handData = hand.CreateOutput(); handData.Update(); // Get number of tracked hands nhands = handData.QueryNumberOfHands(); if (nhands > 0) { // Retrieve hand identifier handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId); // Retrieve hand data handData.QueryHandDataById(handId, out ihand); // Retrieve all hand joint data for (int i = 0; i < nhands; i++) { for (int j = 0; j < 0x20; j++) { PXCMHandData.JointData jointData; ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData); nodes[i][j] = jointData; } } // Get world coordinates for tip of middle finger on the first hand in camera range handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x; handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y; handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z; // Retrieve gesture data if (handData.IsGestureFired("spreadfingers", out gestureData)) { gesture = Gesture.FingerSpread; } else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData)) { gesture = Gesture.Pinch; } else if (handData.IsGestureFired("wave", out gestureData)) { gesture = Gesture.Wave; } } else { gesture = Gesture.Undefined; } // Get alert status for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++) { PXCMHandData.AlertData alertData; if (handData.QueryFiredAlertData(i, out alertData) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } //Displaying last alert switch (alertData.label) { case PXCMHandData.AlertType.ALERT_HAND_DETECTED: detectionAlert = "Hand Detected"; detectionStatusOk = true; break; case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED: detectionAlert = "Hand Not Detected"; detectionStatusOk = false; break; case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED: calibrationAlert = "Hand Calibrated"; calibrationStatusOk = true; break; case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED: calibrationAlert = "Hand Not Calibrated"; calibrationStatusOk = false; break; case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS: bordersAlert = "Hand Inside Borders"; borderStatusOk = true; break; case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS: bordersAlert = "Hand Out Of Borders"; borderStatusOk = false; break; } } UpdateUI(); if (handData != null) handData.Dispose(); } senseManager.ReleaseFrame(); } }
/* Displaying current frame alerts */ private void DisplayAlerts(PXCMHandData handAnalysis, int frameNumber) { bool isChanged = false; string sAlert = "Alert: "; for (int i = 0; i < handAnalysis.QueryFiredAlertsNumber(); i++) { PXCMHandData.AlertData alertData; if (handAnalysis.QueryFiredAlertData(i, out alertData) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } //See PXCMHandAnalysis.AlertData.AlertType for all available alerts switch (alertData.label) { case PXCMHandData.AlertType.ALERT_HAND_DETECTED: { sAlert += "Hand Detected, "; isChanged = true; break; } case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED: { sAlert += "Hand Not Detected, "; isChanged = true; break; } case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED: { sAlert += "Hand Calibrated, "; isChanged = true; break; } case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED: { sAlert += "Hand Not Calibrated, "; isChanged = true; break; } case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS: { sAlert += "Hand Inside Border, "; isChanged = true; break; } case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS: { sAlert += "Hand Out Of Borders, "; isChanged = true; break; } } } if (isChanged) { form.UpdateInfo("Frame " + frameNumber + ") " + sAlert + "\n", Color.RoyalBlue); } }
/* Using PXCMSenseManager to handle data */ public void SimplePipeline() { form.UpdateInfo(String.Empty, Color.Black); bool liveCamera = false; bool flag = true; PXCMSenseManager instance = null; _disconnected = false; instance = form.g_session.CreateSenseManager(); if (instance == null) { form.UpdateStatus("Failed creating SenseManager"); return; } PXCMCaptureManager captureManager = instance.captureManager; if (captureManager != null) { if (form.GetRecordState()) { captureManager.SetFileName(form.GetFileName(), true); PXCMCapture.DeviceInfo info; if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info)) { captureManager.FilterByDeviceInfo(info); } } else if (form.GetPlaybackState()) { captureManager.SetFileName(form.GetFileName(), false); } else { PXCMCapture.DeviceInfo info; if (String.IsNullOrEmpty(form.GetCheckedDevice())) { form.UpdateStatus("Device Failure"); return; } if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info)) { captureManager.FilterByDeviceInfo(info); } liveCamera = true; } } /* Set Module */ pxcmStatus status = instance.EnableHand(form.GetCheckedModule()); PXCMHandModule handAnalysis = instance.QueryHand(); if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null) { form.UpdateStatus("Failed Loading Module"); return; } PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler(); handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame); PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration(); PXCMHandData handData = handAnalysis.CreateOutput(); if (handConfiguration == null) { form.UpdateStatus("Failed Create Configuration"); return; } if (handData == null) { form.UpdateStatus("Failed Create Output"); return; } if (form.getInitGesturesFirstTime() == false) { int totalNumOfGestures = handConfiguration.QueryGesturesTotalNumber(); if (totalNumOfGestures > 0) { this.form.UpdateGesturesToList("", 0); for (int i = 0; i < totalNumOfGestures; i++) { string gestureName = string.Empty; if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) == pxcmStatus.PXCM_STATUS_NO_ERROR) { this.form.UpdateGesturesToList(gestureName, i + 1); } } form.setInitGesturesFirstTime(true); form.UpdateGesturesListSize(); } } FPSTimer timer = new FPSTimer(form); form.UpdateStatus("Init Started"); if (handAnalysis != null && instance.Init(handler) == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMCapture.DeviceInfo dinfo; PXCMCapture.Device device = instance.captureManager.device; if (device != null) { device.QueryDeviceInfo(out dinfo); _maxRange = device.QueryDepthSensorRange().max; } if (handConfiguration != null) { handConfiguration.EnableAllAlerts(); handConfiguration.EnableSegmentationImage(true); handConfiguration.ApplyChanges(); handConfiguration.Update(); } form.UpdateStatus("Streaming"); int frameCounter = 0; int frameNumber = 0; while (!form.stop) { string gestureName = form.GetGestureName(); if (string.IsNullOrEmpty(gestureName) == false) { if (handConfiguration.IsGestureEnabled(gestureName) == false) { handConfiguration.DisableAllGestures(); handConfiguration.EnableGesture(gestureName, true); handConfiguration.ApplyChanges(); } } else { handConfiguration.DisableAllGestures(); handConfiguration.ApplyChanges(); } if (instance.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR) { break; } frameCounter++; if (!DisplayDeviceConnection(!instance.IsConnected())) { if (handData != null) { handData.Update(); } PXCMCapture.Sample sample = instance.QueryHandSample(); if (sample != null && sample.depth != null) { DisplayPicture(sample.depth, handData); if (handData != null) { frameNumber = liveCamera ? frameCounter : instance.captureManager.QueryFrameIndex(); DisplayJoints(handData); DisplayGesture(handData, frameNumber); DisplayAlerts(handData, frameNumber); } form.UpdatePanel(); } timer.Tick(); } instance.ReleaseFrame(); } } else { form.UpdateStatus("Init Failed"); flag = false; } foreach (PXCMImage pxcmImage in m_images) { pxcmImage.Dispose(); } // Clean Up if (handData != null) { handData.Dispose(); } if (handConfiguration != null) { handConfiguration.Dispose(); } instance.Close(); instance.Dispose(); if (flag) { form.UpdateStatus("Stopped"); } }
/* Displaying Depth/Mask Images - for depth image only we use a delay of NumberOfFramesToDelay to sync image with tracking */ private unsafe void DisplayPicture(PXCMImage depth, PXCMHandData handAnalysis) { if (depth == null) { return; } PXCMImage image = depth; //Mask Image if (form.GetLabelmapState()) { Bitmap labeledBitmap = null; try { labeledBitmap = new Bitmap(image.info.width, image.info.height, PixelFormat.Format32bppRgb); for (int j = 0; j < handAnalysis.QueryNumberOfHands(); j++) { int id; PXCMImage.ImageData data; handAnalysis.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, j, out id); //Get hand by time of appearance PXCMHandData.IHand handData; handAnalysis.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, j, out handData); if (handData != null && (handData.QuerySegmentationImage(out image) >= pxcmStatus.PXCM_STATUS_NO_ERROR)) { if (image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { Rectangle rect = new Rectangle(0, 0, image.info.width, image.info.height); BitmapData bitmapdata = labeledBitmap.LockBits(rect, ImageLockMode.ReadWrite, labeledBitmap.PixelFormat); byte *numPtr = (byte *)bitmapdata.Scan0; //dst byte *numPtr2 = (byte *)data.planes[0]; //row int imagesize = image.info.width * image.info.height; byte num2 = (byte)handData.QueryBodySide(); byte tmp = 0; for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++) { tmp = (byte)(LUT[numPtr2[0]] * num2 * 100); numPtr[0] = (Byte)(tmp | numPtr[0]); numPtr[1] = (Byte)(tmp | numPtr[1]); numPtr[2] = (Byte)(tmp | numPtr[2]); numPtr[3] = 0xff; } labeledBitmap.UnlockBits(bitmapdata); image.ReleaseAccess(data); } } } if (labeledBitmap != null) { form.DisplayBitmap(labeledBitmap); labeledBitmap.Dispose(); } image.Dispose(); } catch (Exception) { if (labeledBitmap != null) { labeledBitmap.Dispose(); } if (image != null) { image.Dispose(); } } }//end label image //Depth Image else { //collecting 3 images inside a queue and displaying the oldest image PXCMImage.ImageInfo info; PXCMImage image2; info = image.QueryInfo(); image2 = form.g_session.CreateImage(info); if (image2 == null) { return; } image2.CopyImage(image); m_images.Enqueue(image2); if (m_images.Count == NumberOfFramesToDelay) { Bitmap depthBitmap; try { depthBitmap = new Bitmap(image.info.width, image.info.height, PixelFormat.Format32bppRgb); } catch (Exception) { image.Dispose(); PXCMImage queImage = m_images.Dequeue(); queImage.Dispose(); return; } PXCMImage.ImageData data3; PXCMImage image3 = m_images.Dequeue(); if (image3.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH, out data3) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { float fMaxValue = _maxRange; byte cVal; Rectangle rect = new Rectangle(0, 0, image.info.width, image.info.height); BitmapData bitmapdata = depthBitmap.LockBits(rect, ImageLockMode.ReadWrite, depthBitmap.PixelFormat); byte * pDst = (byte *)bitmapdata.Scan0; short *pSrc = (short *)data3.planes[0]; int size = image.info.width * image.info.height; for (int i = 0; i < size; i++, pSrc++, pDst += 4) { cVal = (byte)((*pSrc) / fMaxValue * 255); if (cVal != 0) { cVal = (byte)(255 - cVal); } pDst[0] = cVal; pDst[1] = cVal; pDst[2] = cVal; pDst[3] = 255; } try { depthBitmap.UnlockBits(bitmapdata); } catch (Exception) { image3.ReleaseAccess(data3); depthBitmap.Dispose(); image3.Dispose(); return; } form.DisplayBitmap(depthBitmap); image3.ReleaseAccess(data3); } depthBitmap.Dispose(); image3.Dispose(); } } }
void OnEnable() { Initialized = false; /* Create a SenseManager instance */ SenseManager = PXCMSenseManager.CreateInstance(); if (SenseManager == null) { print("Unable to create the pipeline instance"); return; } if (_speechCommandsRef.Count != 0) { SetSenseOption(SenseOption.SenseOptionID.Speech); } int numberOfEnabledModalities = 0; //Set mode according to RunMode - play from file / record / live stream if (RunMode == MCTTypes.RunModes.PlayFromFile) { //CHECK IF FILE EXISTS if (!System.IO.File.Exists(FilePath)) { Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream"); RunMode = MCTTypes.RunModes.LiveStream; } else { PXCMCaptureManager cManager = SenseManager.QueryCaptureManager(); cManager.SetFileName(FilePath, false); Debug.Log("SenseToolkitManager: Playing from file: " + FilePath); } } if (RunMode == MCTTypes.RunModes.RecordToFile) { //CHECK IF PATH string PathOnly = FilePath; while (!PathOnly[PathOnly.Length - 1].Equals('\\')) { PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1); } if (!System.IO.Directory.Exists(PathOnly)) { Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream"); RunMode = MCTTypes.RunModes.LiveStream; } else { PXCMCaptureManager cManager = SenseManager.QueryCaptureManager(); cManager.SetFileName(FilePath, true); Debug.Log("SenseToolkitManager: Recording to file: " + FilePath); } } /* Enable modalities according to the set options*/ if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true)) { SenseManager.EnableFace(); _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true; SetSenseOption(SenseOption.SenseOptionID.VideoColorStream); numberOfEnabledModalities++; } if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true)) { _sts = SenseManager.EnableHand(); _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true; numberOfEnabledModalities++; } if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true)) { _sts = SenseManager.EnableBlob(); _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled = true; numberOfEnabledModalities++; } if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true)) { _sts = SenseManager.EnableTracker(); _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true; numberOfEnabledModalities++; } if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true)) { if (!SpeechManager.IsInitialized) { if (SpeechManager.InitalizeSpeech()) { _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true; numberOfEnabledModalities++; } else { UnsetSenseOption(SenseOption.SenseOptionID.Speech); } } else { _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true; numberOfEnabledModalities++; } } if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) || IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true)) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0); _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true; numberOfEnabledModalities++; } if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true)) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0); _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true; numberOfEnabledModalities++; } if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true)) { if (ColorImageQuality == MCTTypes.RGBQuality.FullHD) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0); } else if (ColorImageQuality == MCTTypes.RGBQuality.HD) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0); } else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0); } else { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0); } _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true; numberOfEnabledModalities++; } if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true)) { if (ColorImageQuality == MCTTypes.RGBQuality.FullHD) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0); } else if (ColorImageQuality == MCTTypes.RGBQuality.HD) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0); } else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD) { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0); } else { SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0); } SenseManager.Enable3DSeg(); _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true; _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true; numberOfEnabledModalities++; } /* Initialize the execution */ _sts = SenseManager.Init(); if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { if (numberOfEnabledModalities > 0) { Debug.LogError("Unable to initialize all modalities"); } return; } //Set different configurations: // Face if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true)) { var faceModule = SenseManager.QueryFace(); if (faceModule == null) { throw new UnityException("QueryFace returned null"); } var faceConfiguration = faceModule.CreateActiveConfiguration(); if (faceConfiguration == null) { throw new UnityException("CreateActiveConfiguration returned null"); } faceConfiguration.Update(); faceConfiguration.detection.isEnabled = true; faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED; faceConfiguration.landmarks.isEnabled = true; faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED; faceConfiguration.pose.isEnabled = true; faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED; faceConfiguration.DisableAllAlerts(); faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME; if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15)) { Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1"); NumberOfDetectedFaces = 1; } faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces; faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces; faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces; PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions(); if (expressionConfig == null) { throw new UnityException("QueryExpressions returned null"); } expressionConfig.Enable(); expressionConfig.EnableAllExpressions(); faceConfiguration.ApplyChanges(); faceConfiguration.Dispose(); FaceModuleOutput = faceModule.CreateOutput(); UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream); } // Hand if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true)) { PXCMHandModule handAnalysis = SenseManager.QueryHand(); if (handAnalysis == null) { throw new UnityException("QueryHand returned null"); } PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration(); if (handConfiguration == null) { throw new UnityException("CreateActiveConfiguration returned null"); } handConfiguration.Update(); handConfiguration.EnableAllGestures(); handConfiguration.EnableStabilizer(true); handConfiguration.DisableAllAlerts(); handConfiguration.EnableSegmentationImage(false); handConfiguration.ApplyChanges(); handConfiguration.Dispose(); HandDataOutput = handAnalysis.CreateOutput(); } // Blob if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true)) { PXCMBlobModule blobAnalysis = SenseManager.QueryBlob(); if (blobAnalysis == null) { throw new UnityException("QueryBlob returned null"); } PXCMBlobConfiguration blobConfiguration = blobAnalysis.CreateActiveConfiguration(); if (blobConfiguration == null) { throw new UnityException("CreateActiveConfiguration returned null"); } blobConfiguration.Update(); blobConfiguration.EnableContourExtraction(true); blobConfiguration.EnableSegmentationImage(true); blobConfiguration.EnableStabilizer(true); blobConfiguration.SetMaxDistance(50 * 10); blobConfiguration.ApplyChanges(); blobConfiguration.Dispose(); BlobDataOutput = blobAnalysis.CreateOutput(); } if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true)) { if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true) { _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true; OnDisable(); OnEnable(); } } if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true)) { UpdateSpeechCommands(); SpeechManager.Start(); } // Create an instance for the projection & blob extractor if (Projection == null) { Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection(); } /* GZ * if (BlobExtractor == null) * { * SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor); * }*/ // Set initialization flag Initialized = true; }
public static void start(IoT_RealSense_Surfing frm) { keepLooping = true; iot_form = frm; InitializeMqqtClient(); PXCMSession session = PXCMSession.CreateInstance(); // Querying the SDK version Console.WriteLine(frm.comboTarget.SelectedText); if (session != null) { // Optional steps to send feedback to Intel Corporation to understand how often each SDK sample is used. PXCMMetadata md = session.QueryInstance <PXCMMetadata>(); if (md != null) { string sample_name = "Emotion Viewer CS"; md.AttachBuffer(1297303632, System.Text.Encoding.Unicode.GetBytes(sample_name)); } //Application.Run(new MainForm(session)); //session.Dispose(); } //PXCMSession.ImplVersion version = session.QueryVersion(); //Console.WriteLine("RealSense SDK Version {0}.{1}", version.major, version.minor); session.CreateImpl <PXCMRotation>(out rotationHelper); // Creating the SenseManager PXCMSenseManager senseManager = session.CreateSenseManager(); if (senseManager == null) { Console.WriteLine("Failed to create the SenseManager object."); return; } // Enabling Emotion Module pxcmStatus enablingModuleStatus = senseManager.EnableEmotion(); if (enablingModuleStatus != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("Failed to enable the Emotion Module"); return; } // Getting the instance of the Emotion Module PXCMEmotion emotionModule = senseManager.QueryEmotion(); if (emotionModule == null) { Console.WriteLine("Failed to query the emotion module"); return; } // Initializing the camera if (senseManager.Init() < pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("Failed to initialize the SenseManager"); return; } // Enabling the Hand module pxcmStatus enablingModuleStatus1 = senseManager.EnableHand("Hand Module"); if (enablingModuleStatus1 != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("Failed to enable the Hand Module"); return; } // Getting the instance of the Hand Module PXCMHandModule handModule = senseManager.QueryHand(); if (handModule == null) { Console.WriteLine("Failed to get the HandModule object."); return; } // Creating an active configuration PXCMHandConfiguration handConfiguration = handModule.CreateActiveConfiguration(); if (handConfiguration == null) { Console.WriteLine("Failed to create the HandConfiguration object."); return; } // Listing the available gestures int supportedGesturesCount = handConfiguration.QueryGesturesTotalNumber(); if (supportedGesturesCount > 0) { Console.WriteLine("Supported gestures:"); for (int i = 0; i < supportedGesturesCount; i++) { string gestureName = string.Empty; if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) == pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine("\t" + gestureName); } } } // Enabling some gestures String[] enabledGestures = { GESTURE_CLICK, GESTURE_VSIGN, GESTURE_FIST, GESTURE_SPREADFINGERS }; foreach (String gesture in enabledGestures) { if (!handConfiguration.IsGestureEnabled(gesture)) { handConfiguration.EnableGesture(gesture); } } handConfiguration.ApplyChanges(); // Creating a data output object PXCMHandData handData = handModule.CreateOutput(); if (handData == null) { Console.WriteLine("Failed to create the HandData object."); return; } // Initializing the SenseManager if (senseManager.Init() != pxcmStatus.PXCM_STATUS_NO_ERROR) { Console.WriteLine(senseManager.Init()); return; } // Looping to query the hands information while (keepLooping) { // Acquiring a frame if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR) { break; } // Updating the hand data if (handData != null) { handData.Update(); } //ProcessHands(handData); ProcessGestures(handData); ProcessEmotions(emotionModule); // Releasing the acquired frame senseManager.ReleaseFrame(); /* using another frame to process different stuff? may be... * // Acquiring a frame * if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR) * { * break; * } * * // Processing Emotions * ProcessEmotions(emotionModule); * * // Releasing the acquired frame * senseManager.ReleaseFrame();*/ } // Releasing resources if (handData != null) { handData.Dispose(); } if (handConfiguration != null) { handConfiguration.Dispose(); } rotationHelper.Dispose(); senseManager.Close(); senseManager.Dispose(); session.Dispose(); client.Disconnect(); }
protected virtual void getStatus(PXCMHandData.JointData[] data, bool isLeft){ if (isLeft) lhstatus = checkMotion (data); else rhstatus = checkMotion (data); }
protected virtual Status checkMotion(PXCMHandData.JointData[] data){ if (isPrepared (data)) return Status.Prepared; if (isBad (data)) return Status.Bad; if (isGood (data)) return Status.Good; if (isGreat (data)) return Status.Great; if (isDone (data)) return Status.Done; return Status.None; }
// 手の検出の初期化 private void InitializeHandTracking() { // 手の検出器を取得する handAnalyzer = senseManager.QueryHand(); if ( handAnalyzer == null ) { throw new Exception( "手の検出器の取得に失敗しました" ); } // 手のデータを作成する handData = handAnalyzer.CreateOutput(); if ( handData == null ) { throw new Exception( "手の検出器の作成に失敗しました" ); } // RealSense カメラであれば、プロパティを設定する var device = senseManager.QueryCaptureManager().QueryDevice(); PXCMCapture.DeviceInfo dinfo; device.QueryDeviceInfo( out dinfo ); if ( dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM ) { device.SetDepthConfidenceThreshold( 1 ); //device.SetMirrorMode( PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED ); device.SetIVCAMFilterOption( 6 ); } // 手の検出の設定 var config = handAnalyzer.CreateActiveConfiguration(); config.EnableSegmentationImage( true ); config.ApplyChanges(); config.Update(); }
protected virtual bool isDone(PXCMHandData.JointData[] data){ return false; }
void OnDisable() { //Disposses all modules Initialized = false; if (SenseManager == null) { return; } DisposeFunctions.ForEach(i => i.DynamicInvoke()); if (FaceModuleOutput != null) { FaceModuleOutput.Dispose(); FaceModuleOutput = null; } if (HandDataOutput != null) { SenseManager.PauseHand(true); HandDataOutput.Dispose(); HandDataOutput = null; } if (BlobDataOutput != null) { SenseManager.PauseBlob(true); BlobDataOutput.Dispose(); BlobDataOutput = null; } if (ImageRgbOutput != null) { ImageRgbOutput.Dispose(); ImageRgbOutput = null; } if (ImageDepthOutput != null) { ImageDepthOutput.Dispose(); ImageDepthOutput = null; } if (ImageIROutput != null) { ImageIROutput.Dispose(); ImageIROutput = null; } if (Image3DSegmentationOutput != null) { Image3DSegmentationOutput.Dispose(); Image3DSegmentationOutput = null; } if (Projection != null) { Projection.Dispose(); Projection = null; } /* GZ * if (BlobExtractor != null) * { * BlobExtractor.Dispose(); * BlobExtractor = null; * } */ UvMap = null; PointCloud = null; SenseManager.Dispose(); SenseManager = null; }
void OnFiredGesture(PXCMHandData.GestureData data) { Debug.Log(data.name.ToString()); string gestureName = data.name.ToString(); if (gestureName.Equals("fist")) { gesture = gestureName; } else if (gestureName.Equals("v_sign")) { gesture = gestureName; } else if (gestureName.Equals("thumb_down")) { gesture = gestureName; } else if (gestureName.Equals("spreadfingers")) { gesture = ""; } }
private static void ProcessGestures(PXCMHandData handData) { // Processing gestures // Querying how many gestures were detected int firedGesturesNumber = handData.QueryFiredGesturesNumber(); // Querying which gestures were detected for (int i = 0; i < firedGesturesNumber; i++) { PXCMHandData.GestureData gestureData; if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR) { // Getting the gesture name String gestureName = gestureData.name; // Getting the gesture state PXCMHandData.GestureStateType gestureState = gestureData.state; // Getting the gesture timestamp long timestamp = gestureData.timeStamp; // Getting the hand information about which hand fired the gesture PXCMHandData.IHand handInfo; if (handData.QueryHandDataById(gestureData.handId, out handInfo) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // Getting which hand fired the gesture PXCMHandData.BodySideType bodySideHand = handInfo.QueryBodySide(); if (gestureState == PXCMHandData.GestureStateType.GESTURE_STATE_START) { Console.WriteLine("Gesture \"{0}\" was detected. State:{1}, Timestamp:{2}, BodySide:{3}", gestureName, gestureState, timestamp, bodySideHand); Console.WriteLine("--------------------"); if (gestureName.CompareTo(GESTURE_FIST) == 0) { if (currentGesture.CompareTo(GESTURE_FIST) != 0) { //SendMqttMessage("surfboard2/relay?0"); String target = (String)iot_form.comboTarget.SelectedItem; String hclosed = (String)iot_form.comboHandsClosed.SelectedItem; //if (target.Equals("")) target = "*"; //SendMqttMessage(target + "/" + hclosed); SendMqttMessage(hclosed); currentGesture = GESTURE_FIST; } } else if (gestureName.CompareTo(GESTURE_SPREADFINGERS) == 0) { if (currentGesture.CompareTo(GESTURE_SPREADFINGERS) != 0) { String target = (String)iot_form.comboTarget.SelectedItem; String hopen = (String)iot_form.comboGive5Action.SelectedItem; //if (target.Equals("")) target = "*"; SendMqttMessage(hopen); //SendMqttMessage(target + "/" + hopen); //System.Diagnostics.Process.Start("http://www.globalcode.com.br"); //SendMqttMessage("surfboard2/relay?1"); currentGesture = GESTURE_SPREADFINGERS; } } else if (gestureName.CompareTo(GESTURE_CLICK) == 0) { if (currentGesture.CompareTo(GESTURE_CLICK) != 0) { SendMqttMessage("green?255"); currentGesture = GESTURE_CLICK; } } else if (gestureName.CompareTo(GESTURE_VSIGN) == 0) { if (currentGesture.CompareTo(GESTURE_VSIGN) != 0) { //System.Diagnostics.Process.Start("https://www.youtube.com/watch?v=MVm5hcQYJ-U"); SendMqttMessage("green?0"); keepLooping = false; currentGesture = GESTURE_VSIGN; } } } } } }