public string findVovel(PXCMHandData.JointData[][] nodes, int tip)
{
PXCMPoint3DF32 tipPoint = nodes[1][5 + 4 * tip].positionWorld;
float minTipDistance = 1000;
int tipIndex = -1;
for (int i = 0; i < 5; i++)
{
float tipDistance = Distance(tipPoint, nodes[0][5 + i * 4].positionWorld);
if (tipDistance < minTipDistance)
{
minTipDistance = tipDistance;
tipIndex = i;
}
}
switch (tipIndex)
{
case 0:
return "A";
case 1:
return "E";
case 2:
return "I";
case 3:
return "O";
case 4:
return "U";
default:
return "?";
}
}
protected override bool isBad(PXCMHandData.JointData[] data){
Double DeltaIndex = Math.Sqrt(Math.Pow(data [8].positionWorld.x - data [6].positionWorld.x,2)
+Math.Pow(data [8].positionWorld.z - data [6].positionWorld.z,2));
// myTextLeft.text = DeltaIndex.ToString ();
if (DeltaIndex > MaxDelta) {
return true;
}
Double DeltaRing = Math.Sqrt(Math.Pow(data [16].positionWorld.x - data [14].positionWorld.x,2)
+Math.Pow(data [16].positionWorld.z - data [14].positionWorld.z,2));
// myTextLeft.text += "\n"+DeltaRing.ToString ();
if (DeltaRing > MaxDelta) {
return true;
}
Double DeltaPinky = Math.Sqrt(Math.Pow(data [20].positionWorld.x - data [18].positionWorld.x,2)
+Math.Pow(data [20].positionWorld.z - data [18].positionWorld.z,2));
// myTextLeft.text += "\n"+DeltaRing.ToString ();
if (DeltaPinky > MaxDelta) {
return true;
}
Double angle1 = 2 * Math.Acos(data[11].localRotation.w);
Double angle2 = 2 * Math.Acos (data [12].localRotation.w);
if (angle1 >= 0.5 || (angle2 <= 0.5 && angle2 >= 2))
return true;
return false;
}
public static int JointToInt(PXCMHandData.JointType label)
{
int jointLabel = -1;
switch (label)
{
case PXCMHandData.JointType.JOINT_THUMB_TIP: { jointLabel = 0; break; }
case PXCMHandData.JointType.JOINT_INDEX_BASE: { jointLabel = 1; break; }
case PXCMHandData.JointType.JOINT_MIDDLE_TIP: { jointLabel = 2; break; }
case PXCMHandData.JointType.JOINT_RING_TIP: { jointLabel = 3; break; }
case PXCMHandData.JointType.JOINT_PINKY_TIP: { jointLabel = 4; break; }
case PXCMHandData.JointType.JOINT_WRIST:
case PXCMHandData.JointType.JOINT_CENTER:
case PXCMHandData.JointType.JOINT_THUMB_BASE:
case PXCMHandData.JointType.JOINT_THUMB_JT1:
case PXCMHandData.JointType.JOINT_THUMB_JT2:
case PXCMHandData.JointType.JOINT_INDEX_TIP:
case PXCMHandData.JointType.JOINT_INDEX_JT1:
case PXCMHandData.JointType.JOINT_INDEX_JT2:
case PXCMHandData.JointType.JOINT_MIDDLE_BASE:
case PXCMHandData.JointType.JOINT_MIDDLE_JT1:
case PXCMHandData.JointType.JOINT_MIDDLE_JT2:
case PXCMHandData.JointType.JOINT_RING_BASE:
case PXCMHandData.JointType.JOINT_RING_JT1:
case PXCMHandData.JointType.JOINT_RING_JT2:
case PXCMHandData.JointType.JOINT_PINKY_BASE:
case PXCMHandData.JointType.JOINT_PINKY_JT1:
case PXCMHandData.JointType.JOINT_PINKY_JT2: { jointLabel = -1; break; }
}
return jointLabel;
}
private void TrackIndex(Finger finger, PXCMHandData.IHand handInfo) {
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_BASE, finger.BaseJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_JT1, finger.FirstJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_JT2, finger.SecondJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_TIP, finger);
SetFingerOpenness(finger, PXCMHandData.FingerType.FINGER_INDEX, handInfo);
}
protected override void getStatus(PXCMHandData.JointData[] data, bool isLeft){
if (isLeft)
lhstatus = checkMotion (data);
if (lhstatus != Status.None && lhstatus > BestPerform)
BestPerform = lhstatus;
label1.GetComponent<UILabel>().text = lhstatus.ToString();
}
private void SetHandPosition(Hand hand, PXCMHandData.IHand handInfo) {
var world = handInfo.QueryMassCenterWorld();
if (_smoothers.ContainsKey(hand)) {
world = _smoothers[hand].SmoothValue(world);
}
var imagePosition = ToPoint3D(handInfo.QueryMassCenterImage());
var worldPosition = ToPoint3D(world);
hand.Position = CreatePosition(imagePosition, worldPosition);
}
protected override Status checkMotion(PXCMHandData.JointData[] data){
if (isBad (data))
return Status.Bad;
if (isGreat (data))
return Status.Great;
if (isGood (data))
return Status.Good;
return Status.None;
}
private void OnFiredAlert(PXCMHandData.AlertData alertData)
{
switch (alertData.label)
{
case PXCMHandData.AlertType.ALERT_HAND_DETECTED:
calibrated = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_TRACKED:
calibrated = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_TRACKED:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED:
calibrated = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS:
calibrated = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_LEFT_BORDER:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_RIGHT_BORDER:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_TOP_BORDER:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BOTTOM_BORDER:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_TOO_FAR:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_TOO_CLOSE:
calibrated = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_LOW_CONFIDENCE:
calibrated = false;
break;
default:
calibrated = false;
throw new ArgumentOutOfRangeException();
}
}
private void OnGesture(PXCMHandData.GestureData gesturedata) {
string g = $"Gesture: {gesturedata.name}-{gesturedata.handId}-{gesturedata.state}";
Debug.WriteLine(g);
switch (gesturedata.name) {
case "wave":
//_gestures.OnWave(new GestureEventArgs("wave"));
return;
//case "swipe_left":
// _sensor.OnSlideLeft(new GestureEventArgs("Swipe Left"));
// return;
}
}
private void ExtractHand(PXCMHandData handData, Hand hand, PXCMHandData.AccessOrderType access)
{
PXCMHandData.IHand handInfo;
if (handData.QueryHandData(access, 0, out handInfo) != NoError) {
hand.Visible = false;
return;
}
hand.Visible = true;
var openness = handInfo.QueryOpenness();
hand.Open = openness > 75;
var position = handInfo.QueryMassCenterWorld();
hand.Distance = (int)(position.z * 100);
}
private int weightsNum = 4; //smoothing factor
#endregion Fields
#region Methods
//Display Gestures
void DisplayGestures(PXCMHandData.GestureData gestureData)
{
//if (handList.ContainsKey (gestureData.handId)) {
//switch ((PXCMHandData.BodySideType)handList [gestureData.handId]) {
//case PXCMHandData.BodySideType.BODY_SIDE_LEFT:
// myTextLeft.text = gestureData.name.ToString ();
// break;
//case PXCMHandData.BodySideType.BODY_SIDE_RIGHT:
// myTextRight.text = gestureData.name.ToString ();
// break;
//}
//}
}
private void SetHandOrientation(Hand hand, PXCMHandData.IHand handInfo) {
var d4 = handInfo.QueryPalmOrientation();
PXCMRotation rotationHelper;
_camera.Session.CreateImpl(out rotationHelper);
rotationHelper.SetFromQuaternion(d4);
var rotationEuler = rotationHelper.QueryEulerAngles(PXCMRotation.EulerOrder.PITCH_YAW_ROLL);
var x = rotationEuler.x * 180 / Math.PI;
var y = rotationEuler.y * 180 / Math.PI;
var z = rotationEuler.z * 180 / Math.PI;
hand.Rotation = new Rotation(x, y, z);
rotationHelper.Dispose();
}
private void onFiredGesture(PXCMHandData.GestureData gestureData)
{
if (gestureData.name.CompareTo("spreadfingers") == 0 && this.isPlaying)
{
this.BackColor = Color.Red;
this.TogglePlay();
}
if (gestureData.name.CompareTo("thumb_up") == 0 && !this.isPlaying)
{
this.BackColor = Color.Green;
this.TogglePlay();
}
}
private void TrackHandAndFingers(Hand hand, PXCMHandData data, PXCMHandData.AccessOrderType label) {
PXCMHandData.IHand handInfo;
if (data.QueryHandData(label, 0, out handInfo) != NoError) {
hand.IsVisible = false;
return;
}
hand.IsVisible = true;
SetHandOrientation(hand, handInfo);
SetHandOpenness(hand, handInfo);
SetHandPosition(hand, handInfo);
TrackIndex(hand.Index, handInfo);
TrackMiddle(hand.Middle, handInfo);
TrackRing(hand.Ring, handInfo);
TrackPinky(hand.Pinky, handInfo);
TrackThumb(hand.Thumb, handInfo);
}
public Camera(params PXCMHandConfiguration.OnFiredGestureDelegate[] dlgts)
{
// Create the manager
this._session = PXCMSession.CreateInstance();
this._mngr = this._session.CreateSenseManager();
// streammmm
PXCMVideoModule.DataDesc desc = new PXCMVideoModule.DataDesc();
desc.deviceInfo.streams = PXCMCapture.StreamType.STREAM_TYPE_COLOR | PXCMCapture.StreamType.STREAM_TYPE_DEPTH;
this._mngr.EnableStreams(desc);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, Camera.WIDTH, Camera.HEIGHT, 30);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, Camera.WIDTH, Camera.HEIGHT, 30);
// Hands
this._mngr.EnableHand();
this._hand = this._mngr.QueryHand();
this._handData = this._hand.CreateOutput();
// Hands config
PXCMHandConfiguration conf = this._hand.CreateActiveConfiguration();
conf.EnableGesture("spreadfingers", false);
conf.EnableGesture("thumb_up", false);
// Subscribe hands alerts
conf.EnableAllAlerts();
conf.SubscribeAlert(this.onFiredAlert);
// Subscribe all gestures
foreach (PXCMHandConfiguration.OnFiredGestureDelegate subscriber in dlgts)
{
conf.SubscribeGesture(subscriber);
}
// and the private one for debug
conf.SubscribeGesture(this.onFiredGesture);
// Apply it all
conf.ApplyChanges();
// Set events
this._handler = new PXCMSenseManager.Handler();
this._handler.onModuleProcessedFrame = this.onModuleProcessedFrame;
this._mngr.Init(this._handler);
}
public override void Listen()
{
// attach the controller to the PXCM sensor
_senseManager = Session.CreateSenseManager();
_senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 30);
_senseManager.EnableHand();
_handModule = _senseManager.QueryHand();
_handData = _handModule.CreateOutput();
_handConfiguration = _handModule.CreateActiveConfiguration();
_handConfiguration.SubscribeGesture(_handGestureHandler);
_handConfiguration.SubscribeAlert(_handAlertHandler);
_handConfiguration.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_TRACKED);
_handConfiguration.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_CALIBRATED);
_handConfiguration.EnableGesture("full_pinch");
_handConfiguration.EnableGesture("thumb_up");
_handConfiguration.ApplyChanges();
_senseManager.Init(_handler);
sensorActive = true;
_senseManager.StreamFrames(true);
_senseManager.Close();
}
/* Displaying current frame gestures */
private void DisplayGesture(PXCMHandData handAnalysis,int frameNumber)
{
int firedGesturesNumber = handAnalysis.QueryFiredGesturesNumber();
string gestureStatusLeft = string.Empty;
string gestureStatusRight = string.Empty;
if (firedGesturesNumber == 0)
{
return;
}
for (int i = 0; i < firedGesturesNumber; i++)
{
PXCMHandData.GestureData gestureData;
if (handAnalysis.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMHandData.IHand handData;
if (handAnalysis.QueryHandDataById(gestureData.handId, out handData) != pxcmStatus.PXCM_STATUS_NO_ERROR)
return;
PXCMHandData.BodySideType bodySideType = handData.QueryBodySide();
if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
gestureStatusLeft += "Left Hand Gesture: " + gestureData.name;
}
else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
gestureStatusRight += "Right Hand Gesture: " + gestureData.name;
}
}
}
if (gestureStatusLeft == String.Empty)
form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusRight + "\n", Color.SeaGreen);
else
form.UpdateInfo("Frame " + frameNumber + ") " + gestureStatusLeft + ", " + gestureStatusRight + "\n", Color.SeaGreen);
}
public static string AlertToString(PXCMHandData.AlertType label)
{
string alertLabel = "";
switch (label)
{
case PXCMHandData.AlertType.ALERT_HAND_DETECTED: { alertLabel = "ALERT_HAND_DETECTED"; break; }
case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED: { alertLabel = "ALERT_HAND_NOT_DETECTED"; break; }
case PXCMHandData.AlertType.ALERT_HAND_TRACKED: { alertLabel = "ALERT_HAND_TRACKED"; break; }
case PXCMHandData.AlertType.ALERT_HAND_NOT_TRACKED: { alertLabel = "ALERT_HAND_NOT_TRACKED"; break; }
case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED: { alertLabel = "ALERT_HAND_CALIBRATED"; break; }
case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED: { alertLabel = "ALERT_HAND_NOT_CALIBRATED"; break; }
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS: { alertLabel = "ALERT_HAND_OUT_OF_BORDERS"; break; }
case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS: { alertLabel = "ALERT_HAND_INSIDE_BORDERS"; break; }
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_LEFT_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_LEFT_BORDER"; break; }
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_RIGHT_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_RIGHT_BORDER"; break; }
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_TOP_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_TOP_BORDER"; break; }
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BOTTOM_BORDER: { alertLabel = "ALERT_HAND_OUT_OF_BOTTOM_BORDER"; break; }
case PXCMHandData.AlertType.ALERT_HAND_TOO_FAR: { alertLabel = "ALERT_HAND_TOO_FAR"; break; }
case PXCMHandData.AlertType.ALERT_HAND_TOO_CLOSE: { alertLabel = "ALERT_HAND_TOO_CLOSE"; break; }
case PXCMHandData.AlertType.ALERT_HAND_LOW_CONFIDENCE: { alertLabel = "ALERT_HAND_LOW_CONFIDENCE"; break; }
}
return alertLabel;
}
private void OnFiredGesture(PXCMHandData.GestureData gestureData)
{
switch (gestureData.state)
{
case PXCMHandData.GestureStateType.GESTURE_STATE_START:
break;
case PXCMHandData.GestureStateType.GESTURE_STATE_IN_PROGRESS:
break;
case PXCMHandData.GestureStateType.GESTURE_STATE_END:
break;
default:
throw new ArgumentOutOfRangeException();
}
if (gestureData.name == "full_pinch")
{
_movementController.ActivateGripper();
}
if (gestureData.name == "thumb_up")
{
_movementController.BlinkLight();
}
}
public HandsRecognition(int mod)
{
try
{
session = PXCMSession.CreateInstance();
mode = mod;
if (mod == 0)
{
rightcp = new checkPiano();
leftcp = new checkPiano();
}
else if (mod == 1)
{
rightcp = new checkDrum();
leftcp = new checkDrum();
}
_disconnected = false;
instance = session.CreateSenseManager();
if (instance == null)
{
MessageBox.Show("Failed creating SenseManager", "OnAlert");
return;
}
/* Set Module */
pxcmStatus status = instance.EnableHand();//form.GetCheckedModule());
handAnalysis = instance.QueryHand();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null)
{
MessageBox.Show("Failed Loading Module", "OnAlert");
return;
}
handler = new PXCMSenseManager.Handler();
handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
handData = handAnalysis.CreateOutput();
}
catch
{
MessageBox.Show("Init Failed.");
Environment.Exit(0);
}
}
/* Displaying current frames hand joints */
private void DisplayJoints(PXCMHandData handOutput, long timeStamp = 0)
{
//Iterate hands
PXCMHandData.JointData[][] nodes = new PXCMHandData.JointData[][] { new PXCMHandData.JointData[0x20], new PXCMHandData.JointData[0x20] };
int numOfHands = handOutput.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
//Get hand by time of appearence
PXCMHandData.IHand handData;
if (handOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, i, out handData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (handData != null)
{
Coordinate coor;
string handSide = "Unknown Hand";
handSide = handData.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT ? "Left Hand" : "Right Hand";
switch (handSide)
{
case ("Left Hand"): { coor.handside = 0; Rnum++; break; }
case ("Right Hand"): { coor.handside = 1; Lnum++; break; }
default: { coor.handside = 2; break; }
}
//Iterate Joints
for (int j = 0; j < 0x20; j++)
{
sr = File.CreateText(FILE_NAME);
PXCMHandData.JointData jointData;
handData.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
nodes[i][j] = jointData;
if (handData.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData) == pxcmStatus.PXCM_STATUS_NO_ERROR && Handsinfo.JointToInt((PXCMHandData.JointType)j) != -1)
{
coor.finger = Handsinfo.JointToInt((PXCMHandData.JointType)j);
coor.x = jointData.positionWorld.x;
coor.y = jointData.positionWorld.y;
coor.z = jointData.positionWorld.z;
//right
if (coor.finger == 1 && coor.handside == 1)
{
this.RHandMove(coor.x, coor.y);
sr.WriteLine(coor.x + " " + coor.y + " " + coor.z + "\n");
MYPoint p = new MYPoint(coor.x, coor.y, coor.z);
int num = rightcp.checkNote(p);
if (num != 0 && Rnum < 2 && Lnum < 2)
{
DateTime DateTime2 = DateTime.Now;
diff = tim.ExecTimeDiff(starttmp, DateTime2);
this.HandClick(num, (int)diff);
}
}
//left
if (coor.finger == 1 && coor.handside == 0)
{
this.LHandMove(coor.x, coor.y);
sr.WriteLine(coor.x + " " + coor.y + " " + coor.z + "\n");
MYPoint p = new MYPoint(coor.x, coor.y, coor.z);
int num = leftcp.checkNote(p);
if (num != 0 && Rnum < 2 && Lnum < 2)
{
DateTime DateTime2 = DateTime.Now;
diff = tim.ExecTimeDiff(starttmp, DateTime2);
this.HandClick(num, (int)diff);
}
}
}
sr.Close();
} // end iterating over joints
}
} // end itrating over hands
}
Lnum = 0; Rnum = 0;
}
private static void ProcessHands(PXCMHandData handData)
{
// Querying how many hands were detected
int numberOfHands = handData.QueryNumberOfHands();
//Console.WriteLine("{0} hand(s) were detected.", numberOfHands);
// Querying the information about detected hands
for (int i = 0; i < numberOfHands; i++)
{
// Querying hand id
int handId;
pxcmStatus queryHandIdStatus = handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, i, out handId);
if (queryHandIdStatus != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("Failed to query the hand Id.");
continue;
}
//Console.WriteLine("Hand id: {0}", handId);
// Querying the hand data
PXCMHandData.IHand hand;
pxcmStatus queryHandStatus = handData.QueryHandDataById(handId, out hand);
if (queryHandStatus == pxcmStatus.PXCM_STATUS_NO_ERROR && hand != null)
{
// Querying the body side (Left/Right)
PXCMHandData.BodySideType bodySide = hand.QueryBodySide();
//Console.WriteLine("Body Side: {0}", bodySide);
// Querying the hand openness
int handOpenness = hand.QueryOpenness();
//Console.WriteLine("Hand openness: {0}", handOpenness);
SendMqttMessage(String.Format("red?{0}", (255 * handOpenness) / 100));
// Querying Hand 2D Position
PXCMPointF32 massCenterImage = hand.QueryMassCenterImage();
//Console.WriteLine("Hand position on image: {0} | {1}", massCenterImage.x, massCenterImage.y);
PXCMPoint4DF32 palmOrientation = hand.QueryPalmOrientation();
rotationHelper.SetFromQuaternion(palmOrientation);
PXCMPoint3DF32 rotationEuler = rotationHelper.QueryEulerAngles();
double angleInDegrees = (180 * rotationEuler.y / Math.PI) + 180;
Console.WriteLine("Angle in degrees: {0}", angleInDegrees);
SendMqttMessage(String.Format("red?{0}", (int)angleInDegrees));
// Console.WriteLine("Rotation x:{0},y:{1},z:{2}", rotationEuler.x, rotationEuler.y, rotationEuler.z);
// Querying Hand 3D Position
PXCMPoint3DF32 massCenterWorld = hand.QueryMassCenterWorld();
//Console.WriteLine("Hand position on world: {0} | {1} | {2}", massCenterWorld.x, massCenterWorld.y, massCenterWorld.z);
/*
* // Querying Hand Joints
* if (hand.HasTrackedJoints())
* {
* foreach (PXCMHandData.JointType jointType in Enum.GetValues(typeof(PXCMHandData.JointType)))
* {
* PXCMHandData.JointData jointData;
* pxcmStatus queryStatus = hand.QueryTrackedJoint(jointType, out jointData);
*
* if (queryStatus == pxcmStatus.PXCM_STATUS_NO_ERROR && jointData != null)
* {
* // Printing joint label and tracking confidence
* Console.WriteLine("Joint {0} with confidence {1}", jointType, jointData.confidence);
*
* // Printing the 2D position (image)
* Console.WriteLine("\t2D Position: {0} | {1}", jointData.positionImage.x, jointData.positionImage.y);
*
* // Printing the 3D position (depth)
* Console.WriteLine("\t3D Position: {0} | {1} | {2}", jointData.positionWorld.x, jointData.positionWorld.y, jointData.positionWorld.z);
* }
* }
* }*/
}
//Console.WriteLine("----------");
//Console.Clear();
}
}
protected override bool isGreat(PXCMHandData.JointData[] data){
Double angle2 = 2 * Math.Acos (data [12].localRotation.w);
if (angle2 > 1.4 && angle2 < 1.6)
return true;
return false;
}
// Update is called once per frame
void Update()
{
/* Make sure SenseManager Instance is valid */
if (sm == null)
{
return;
}
/* Wait until any frame data is available */
if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve hand tracking Module Instance */
handAnalyzer = sm.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput();
if (_handData != null)
{
_handData.Update();
/* Retrieve all joint Data */
bool[] someJointsDetected = { false, false };
for (int i = 0; i < _handData.QueryNumberOfHands(); i++)
{
PXCMHandData.IHand _iHand;
if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
for (int j = 0; j < MaxJoints; j++)
{
if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i, j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
jointData[i, j] = null;
}
else if (jointData[i, j].confidence == 100)
{
someJointsDetected[i] = true;
}
}
if (!handList.ContainsKey(_iHand.QueryUniqueId()))
{
handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide());
}
}
}
for (int i = 0; i < MaxHands; i++)
{
if (!someJointsDetected[i] && handWasTracked[i])
{
for (int j = 0; j < MaxJoints; j++)
{
Joint joint = myJoints[i, j];
if (joint == null)
{
continue;
}
joint.transform.position = joint.originalPosition;
joint.transform.rotation = joint.originalRotation;
foreach (Joint e in joint.extensions)
{
e.transform.position = e.originalPosition;
e.transform.rotation = e.originalRotation;
}
}
}
else
{
if (!handWasTracked[i])
{
if (myJoints[i, 0] == null || jointData[i, 0] == null || jointData[i, 0].confidence < 100)
{
break;
}
PXCMPoint3DF32 smoothedPoint = smoother3D[i, 0].SmoothValue(jointData[i, 0].positionWorld);
Vector3 targetPosition = new Vector3(-1 * smoothedPoint.x, smoothedPoint.y, smoothedPoint.z);
trackingOffset[i] = targetPosition - myJoints[i, 0].originalPosition;
}
/* Smooth the data and move the joints*/
MoveJoints(i);
}
handWasTracked[i] = someJointsDetected[i];
}
}
handAnalyzer.Dispose();
}
sm.ReleaseFrame();
RotateCam();
}
public void DisplayJoints(PXCMHandData.JointData[][] nodes, int numOfHands)
{
if (bitmap == null) return;
if (nodes == null) return;
if (Joints.Checked || Skeleton.Checked)
{
lock (this)
{
int scaleFactor = 1;
Graphics g = Graphics.FromImage(bitmap);
using (Pen boneColor = new Pen(Color.DodgerBlue, 3.0f))
{
for (int i = 0; i < numOfHands; i++)
{
if (nodes[i][0] == null) continue;
int baseX = (int) nodes[i][0].positionImage.x/scaleFactor;
int baseY = (int) nodes[i][0].positionImage.y/scaleFactor;
int wristX = (int) nodes[i][0].positionImage.x/scaleFactor;
int wristY = (int) nodes[i][0].positionImage.y/scaleFactor;
if (Skeleton.Checked)
{
for (int j = 1; j < 22; j++)
{
if (nodes[i][j] == null) continue;
int x = (int) nodes[i][j].positionImage.x/scaleFactor;
int y = (int) nodes[i][j].positionImage.y/scaleFactor;
if (nodes[i][j].confidence <= 0) continue;
if (j == 2 || j == 6 || j == 10 || j == 14 || j == 18)
{
baseX = wristX;
baseY = wristY;
}
g.DrawLine(boneColor, new Point(baseX, baseY), new Point(x, y));
baseX = x;
baseY = y;
}
}
if (Joints.Checked)
{
using (
Pen red = new Pen(Color.Red, 3.0f),
black = new Pen(Color.Black, 3.0f),
green = new Pen(Color.Green, 3.0f),
blue = new Pen(Color.Blue, 3.0f),
cyan = new Pen(Color.Cyan, 3.0f),
yellow = new Pen(Color.Yellow, 3.0f),
orange = new Pen(Color.Orange, 3.0f))
{
Pen currnetPen = black;
for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++)
{
float sz = 4;
if (Labelmap.Checked)
sz = 2;
int x = (int) nodes[i][j].positionImage.x/scaleFactor;
int y = (int) nodes[i][j].positionImage.y/scaleFactor;
if (nodes[i][j].confidence <= 0) continue;
//Wrist
if (j == 0)
{
currnetPen = black;
}
//Center
if (j == 1)
{
currnetPen = red;
sz += 4;
}
//Thumb
if (j == 2 || j == 3 || j == 4 || j == 5)
{
currnetPen = green;
}
//Index Finger
if (j == 6 || j == 7 || j == 8 || j == 9)
{
currnetPen = blue;
}
//Finger
if (j == 10 || j == 11 || j == 12 || j == 13)
{
currnetPen = yellow;
}
//Ring Finger
if (j == 14 || j == 15 || j == 16 || j == 17)
{
currnetPen = cyan;
}
//Pinkey
if (j == 18 || j == 19 || j == 20 || j == 21)
{
currnetPen = orange;
}
if (j == 5 || j == 9 || j == 13 || j == 17 || j == 21)
{
sz += 4;
//currnetPen.Width = 1;
}
g.DrawEllipse(currnetPen, x - sz/2, y - sz/2, sz, sz);
}
}
}
}
}
g.Dispose();
}
}
}
protected virtual bool isPrepared(PXCMHandData.JointData[] data){
return false;
}
private void ProcessingHandThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
hand = senseManager.QueryHand();
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
// Get number of tracked hands
nhands = handData.QueryNumberOfHands();
if (nhands > 0)
{
// Retrieve hand identifier
handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId);
// Retrieve hand data
handData.QueryHandDataById(handId, out ihand);
// Retrieve all hand joint data
for (int i = 0; i < nhands; i++)
{
for (int j = 0; j < 0x20; j++)
{
PXCMHandData.JointData jointData;
ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
nodes[i][j] = jointData;
}
}
// Get world coordinates for tip of middle finger on the first hand in camera range
handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x;
handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y;
handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z;
// Retrieve gesture data
if (handData.IsGestureFired("spreadfingers", out gestureData))
{
gesture = Gesture.FingerSpread;
}
else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
gesture = Gesture.Pinch;
}
else if (handData.IsGestureFired("wave", out gestureData))
{
gesture = Gesture.Wave;
}
}
else
{
gesture = Gesture.Undefined;
}
// Get alert status
for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++)
{
PXCMHandData.AlertData alertData;
if (handData.QueryFiredAlertData(i, out alertData) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; }
//Displaying last alert
switch (alertData.label)
{
case PXCMHandData.AlertType.ALERT_HAND_DETECTED:
detectionAlert = "Hand Detected";
detectionStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED:
detectionAlert = "Hand Not Detected";
detectionStatusOk = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED:
calibrationAlert = "Hand Calibrated";
calibrationStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED:
calibrationAlert = "Hand Not Calibrated";
calibrationStatusOk = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS:
bordersAlert = "Hand Inside Borders";
borderStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS:
bordersAlert = "Hand Out Of Borders";
borderStatusOk = false;
break;
}
}
UpdateUI();
if (handData != null) handData.Dispose();
}
senseManager.ReleaseFrame();
}
}
/* Displaying current frame alerts */
private void DisplayAlerts(PXCMHandData handAnalysis, int frameNumber)
{
bool isChanged = false;
string sAlert = "Alert: ";
for (int i = 0; i < handAnalysis.QueryFiredAlertsNumber(); i++)
{
PXCMHandData.AlertData alertData;
if (handAnalysis.QueryFiredAlertData(i, out alertData) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
//See PXCMHandAnalysis.AlertData.AlertType for all available alerts
switch (alertData.label)
{
case PXCMHandData.AlertType.ALERT_HAND_DETECTED:
{
sAlert += "Hand Detected, ";
isChanged = true;
break;
}
case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED:
{
sAlert += "Hand Not Detected, ";
isChanged = true;
break;
}
case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED:
{
sAlert += "Hand Calibrated, ";
isChanged = true;
break;
}
case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED:
{
sAlert += "Hand Not Calibrated, ";
isChanged = true;
break;
}
case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS:
{
sAlert += "Hand Inside Border, ";
isChanged = true;
break;
}
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS:
{
sAlert += "Hand Out Of Borders, ";
isChanged = true;
break;
}
}
}
if (isChanged)
{
form.UpdateInfo("Frame " + frameNumber + ") " + sAlert + "\n", Color.RoyalBlue);
}
}
/* Using PXCMSenseManager to handle data */
public void SimplePipeline()
{
form.UpdateInfo(String.Empty, Color.Black);
bool liveCamera = false;
bool flag = true;
PXCMSenseManager instance = null;
_disconnected = false;
instance = form.g_session.CreateSenseManager();
if (instance == null)
{
form.UpdateStatus("Failed creating SenseManager");
return;
}
PXCMCaptureManager captureManager = instance.captureManager;
if (captureManager != null)
{
if (form.GetRecordState())
{
captureManager.SetFileName(form.GetFileName(), true);
PXCMCapture.DeviceInfo info;
if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info))
{
captureManager.FilterByDeviceInfo(info);
}
}
else if (form.GetPlaybackState())
{
captureManager.SetFileName(form.GetFileName(), false);
}
else
{
PXCMCapture.DeviceInfo info;
if (String.IsNullOrEmpty(form.GetCheckedDevice()))
{
form.UpdateStatus("Device Failure");
return;
}
if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info))
{
captureManager.FilterByDeviceInfo(info);
}
liveCamera = true;
}
}
/* Set Module */
pxcmStatus status = instance.EnableHand(form.GetCheckedModule());
PXCMHandModule handAnalysis = instance.QueryHand();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null)
{
form.UpdateStatus("Failed Loading Module");
return;
}
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler();
handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
PXCMHandData handData = handAnalysis.CreateOutput();
if (handConfiguration == null)
{
form.UpdateStatus("Failed Create Configuration");
return;
}
if (handData == null)
{
form.UpdateStatus("Failed Create Output");
return;
}
if (form.getInitGesturesFirstTime() == false)
{
int totalNumOfGestures = handConfiguration.QueryGesturesTotalNumber();
if (totalNumOfGestures > 0)
{
this.form.UpdateGesturesToList("", 0);
for (int i = 0; i < totalNumOfGestures; i++)
{
string gestureName = string.Empty;
if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) ==
pxcmStatus.PXCM_STATUS_NO_ERROR)
{
this.form.UpdateGesturesToList(gestureName, i + 1);
}
}
form.setInitGesturesFirstTime(true);
form.UpdateGesturesListSize();
}
}
FPSTimer timer = new FPSTimer(form);
form.UpdateStatus("Init Started");
if (handAnalysis != null && instance.Init(handler) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.DeviceInfo dinfo;
PXCMCapture.Device device = instance.captureManager.device;
if (device != null)
{
device.QueryDeviceInfo(out dinfo);
_maxRange = device.QueryDepthSensorRange().max;
}
if (handConfiguration != null)
{
handConfiguration.EnableAllAlerts();
handConfiguration.EnableSegmentationImage(true);
handConfiguration.ApplyChanges();
handConfiguration.Update();
}
form.UpdateStatus("Streaming");
int frameCounter = 0;
int frameNumber = 0;
while (!form.stop)
{
string gestureName = form.GetGestureName();
if (string.IsNullOrEmpty(gestureName) == false)
{
if (handConfiguration.IsGestureEnabled(gestureName) == false)
{
handConfiguration.DisableAllGestures();
handConfiguration.EnableGesture(gestureName, true);
handConfiguration.ApplyChanges();
}
}
else
{
handConfiguration.DisableAllGestures();
handConfiguration.ApplyChanges();
}
if (instance.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
break;
}
frameCounter++;
if (!DisplayDeviceConnection(!instance.IsConnected()))
{
if (handData != null)
{
handData.Update();
}
PXCMCapture.Sample sample = instance.QueryHandSample();
if (sample != null && sample.depth != null)
{
DisplayPicture(sample.depth, handData);
if (handData != null)
{
frameNumber = liveCamera ? frameCounter : instance.captureManager.QueryFrameIndex();
DisplayJoints(handData);
DisplayGesture(handData, frameNumber);
DisplayAlerts(handData, frameNumber);
}
form.UpdatePanel();
}
timer.Tick();
}
instance.ReleaseFrame();
}
}
else
{
form.UpdateStatus("Init Failed");
flag = false;
}
foreach (PXCMImage pxcmImage in m_images)
{
pxcmImage.Dispose();
}
// Clean Up
if (handData != null)
{
handData.Dispose();
}
if (handConfiguration != null)
{
handConfiguration.Dispose();
}
instance.Close();
instance.Dispose();
if (flag)
{
form.UpdateStatus("Stopped");
}
}
/* Displaying Depth/Mask Images - for depth image only we use a delay of NumberOfFramesToDelay to sync image with tracking */
private unsafe void DisplayPicture(PXCMImage depth, PXCMHandData handAnalysis)
{
if (depth == null)
{
return;
}
PXCMImage image = depth;
//Mask Image
if (form.GetLabelmapState())
{
Bitmap labeledBitmap = null;
try
{
labeledBitmap = new Bitmap(image.info.width, image.info.height, PixelFormat.Format32bppRgb);
for (int j = 0; j < handAnalysis.QueryNumberOfHands(); j++)
{
int id;
PXCMImage.ImageData data;
handAnalysis.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, j, out id);
//Get hand by time of appearance
PXCMHandData.IHand handData;
handAnalysis.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, j, out handData);
if (handData != null &&
(handData.QuerySegmentationImage(out image) >= pxcmStatus.PXCM_STATUS_NO_ERROR))
{
if (image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8,
out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Rectangle rect = new Rectangle(0, 0, image.info.width, image.info.height);
BitmapData bitmapdata = labeledBitmap.LockBits(rect, ImageLockMode.ReadWrite,
labeledBitmap.PixelFormat);
byte *numPtr = (byte *)bitmapdata.Scan0; //dst
byte *numPtr2 = (byte *)data.planes[0]; //row
int imagesize = image.info.width * image.info.height;
byte num2 = (byte)handData.QueryBodySide();
byte tmp = 0;
for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++)
{
tmp = (byte)(LUT[numPtr2[0]] * num2 * 100);
numPtr[0] = (Byte)(tmp | numPtr[0]);
numPtr[1] = (Byte)(tmp | numPtr[1]);
numPtr[2] = (Byte)(tmp | numPtr[2]);
numPtr[3] = 0xff;
}
labeledBitmap.UnlockBits(bitmapdata);
image.ReleaseAccess(data);
}
}
}
if (labeledBitmap != null)
{
form.DisplayBitmap(labeledBitmap);
labeledBitmap.Dispose();
}
image.Dispose();
}
catch (Exception)
{
if (labeledBitmap != null)
{
labeledBitmap.Dispose();
}
if (image != null)
{
image.Dispose();
}
}
}//end label image
//Depth Image
else
{
//collecting 3 images inside a queue and displaying the oldest image
PXCMImage.ImageInfo info;
PXCMImage image2;
info = image.QueryInfo();
image2 = form.g_session.CreateImage(info);
if (image2 == null)
{
return;
}
image2.CopyImage(image);
m_images.Enqueue(image2);
if (m_images.Count == NumberOfFramesToDelay)
{
Bitmap depthBitmap;
try
{
depthBitmap = new Bitmap(image.info.width, image.info.height, PixelFormat.Format32bppRgb);
}
catch (Exception)
{
image.Dispose();
PXCMImage queImage = m_images.Dequeue();
queImage.Dispose();
return;
}
PXCMImage.ImageData data3;
PXCMImage image3 = m_images.Dequeue();
if (image3.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH, out data3) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
float fMaxValue = _maxRange;
byte cVal;
Rectangle rect = new Rectangle(0, 0, image.info.width, image.info.height);
BitmapData bitmapdata = depthBitmap.LockBits(rect, ImageLockMode.ReadWrite, depthBitmap.PixelFormat);
byte * pDst = (byte *)bitmapdata.Scan0;
short *pSrc = (short *)data3.planes[0];
int size = image.info.width * image.info.height;
for (int i = 0; i < size; i++, pSrc++, pDst += 4)
{
cVal = (byte)((*pSrc) / fMaxValue * 255);
if (cVal != 0)
{
cVal = (byte)(255 - cVal);
}
pDst[0] = cVal;
pDst[1] = cVal;
pDst[2] = cVal;
pDst[3] = 255;
}
try
{
depthBitmap.UnlockBits(bitmapdata);
}
catch (Exception)
{
image3.ReleaseAccess(data3);
depthBitmap.Dispose();
image3.Dispose();
return;
}
form.DisplayBitmap(depthBitmap);
image3.ReleaseAccess(data3);
}
depthBitmap.Dispose();
image3.Dispose();
}
}
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
_sts = SenseManager.EnableBlob();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) ||
IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
SenseManager.Enable3DSeg();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (numberOfEnabledModalities > 0)
{
Debug.LogError("Unable to initialize all modalities");
}
return;
}
//Set different configurations:
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
if (faceModule == null)
{
throw new UnityException("QueryFace returned null");
}
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15))
{
Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1");
NumberOfDetectedFaces = 1;
}
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
if (expressionConfig == null)
{
throw new UnityException("QueryExpressions returned null");
}
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
if (handAnalysis == null)
{
throw new UnityException("QueryHand returned null");
}
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableStabilizer(true);
handConfiguration.DisableAllAlerts();
handConfiguration.EnableSegmentationImage(false);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
// Blob
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
PXCMBlobModule blobAnalysis = SenseManager.QueryBlob();
if (blobAnalysis == null)
{
throw new UnityException("QueryBlob returned null");
}
PXCMBlobConfiguration blobConfiguration = blobAnalysis.CreateActiveConfiguration();
if (blobConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
blobConfiguration.Update();
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.EnableStabilizer(true);
blobConfiguration.SetMaxDistance(50 * 10);
blobConfiguration.ApplyChanges();
blobConfiguration.Dispose();
BlobDataOutput = blobAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection & blob extractor
if (Projection == null)
{
Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
/* GZ
* if (BlobExtractor == null)
* {
* SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor);
* }*/
// Set initialization flag
Initialized = true;
}
public static void start(IoT_RealSense_Surfing frm)
{
keepLooping = true;
iot_form = frm;
InitializeMqqtClient();
PXCMSession session = PXCMSession.CreateInstance();
// Querying the SDK version
Console.WriteLine(frm.comboTarget.SelectedText);
if (session != null)
{
// Optional steps to send feedback to Intel Corporation to understand how often each SDK sample is used.
PXCMMetadata md = session.QueryInstance <PXCMMetadata>();
if (md != null)
{
string sample_name = "Emotion Viewer CS";
md.AttachBuffer(1297303632, System.Text.Encoding.Unicode.GetBytes(sample_name));
}
//Application.Run(new MainForm(session));
//session.Dispose();
}
//PXCMSession.ImplVersion version = session.QueryVersion();
//Console.WriteLine("RealSense SDK Version {0}.{1}", version.major, version.minor);
session.CreateImpl <PXCMRotation>(out rotationHelper);
// Creating the SenseManager
PXCMSenseManager senseManager = session.CreateSenseManager();
if (senseManager == null)
{
Console.WriteLine("Failed to create the SenseManager object.");
return;
}
// Enabling Emotion Module
pxcmStatus enablingModuleStatus = senseManager.EnableEmotion();
if (enablingModuleStatus != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("Failed to enable the Emotion Module");
return;
}
// Getting the instance of the Emotion Module
PXCMEmotion emotionModule = senseManager.QueryEmotion();
if (emotionModule == null)
{
Console.WriteLine("Failed to query the emotion module");
return;
}
// Initializing the camera
if (senseManager.Init() < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("Failed to initialize the SenseManager");
return;
}
// Enabling the Hand module
pxcmStatus enablingModuleStatus1 = senseManager.EnableHand("Hand Module");
if (enablingModuleStatus1 != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("Failed to enable the Hand Module");
return;
}
// Getting the instance of the Hand Module
PXCMHandModule handModule = senseManager.QueryHand();
if (handModule == null)
{
Console.WriteLine("Failed to get the HandModule object.");
return;
}
// Creating an active configuration
PXCMHandConfiguration handConfiguration = handModule.CreateActiveConfiguration();
if (handConfiguration == null)
{
Console.WriteLine("Failed to create the HandConfiguration object.");
return;
}
// Listing the available gestures
int supportedGesturesCount = handConfiguration.QueryGesturesTotalNumber();
if (supportedGesturesCount > 0)
{
Console.WriteLine("Supported gestures:");
for (int i = 0; i < supportedGesturesCount; i++)
{
string gestureName = string.Empty;
if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("\t" + gestureName);
}
}
}
// Enabling some gestures
String[] enabledGestures = { GESTURE_CLICK, GESTURE_VSIGN, GESTURE_FIST, GESTURE_SPREADFINGERS };
foreach (String gesture in enabledGestures)
{
if (!handConfiguration.IsGestureEnabled(gesture))
{
handConfiguration.EnableGesture(gesture);
}
}
handConfiguration.ApplyChanges();
// Creating a data output object
PXCMHandData handData = handModule.CreateOutput();
if (handData == null)
{
Console.WriteLine("Failed to create the HandData object.");
return;
}
// Initializing the SenseManager
if (senseManager.Init() != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine(senseManager.Init());
return;
}
// Looping to query the hands information
while (keepLooping)
{
// Acquiring a frame
if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
break;
}
// Updating the hand data
if (handData != null)
{
handData.Update();
}
//ProcessHands(handData);
ProcessGestures(handData);
ProcessEmotions(emotionModule);
// Releasing the acquired frame
senseManager.ReleaseFrame();
/* using another frame to process different stuff? may be...
* // Acquiring a frame
* if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
* {
* break;
* }
*
* // Processing Emotions
* ProcessEmotions(emotionModule);
*
* // Releasing the acquired frame
* senseManager.ReleaseFrame();*/
}
// Releasing resources
if (handData != null)
{
handData.Dispose();
}
if (handConfiguration != null)
{
handConfiguration.Dispose();
}
rotationHelper.Dispose();
senseManager.Close();
senseManager.Dispose();
session.Dispose();
client.Disconnect();
}
protected virtual void getStatus(PXCMHandData.JointData[] data, bool isLeft){
if (isLeft)
lhstatus = checkMotion (data);
else
rhstatus = checkMotion (data);
}
protected virtual Status checkMotion(PXCMHandData.JointData[] data){
if (isPrepared (data))
return Status.Prepared;
if (isBad (data))
return Status.Bad;
if (isGood (data))
return Status.Good;
if (isGreat (data))
return Status.Great;
if (isDone (data))
return Status.Done;
return Status.None;
}
// 手の検出の初期化
private void InitializeHandTracking()
{
// 手の検出器を取得する
handAnalyzer = senseManager.QueryHand();
if ( handAnalyzer == null ) {
throw new Exception( "手の検出器の取得に失敗しました" );
}
// 手のデータを作成する
handData = handAnalyzer.CreateOutput();
if ( handData == null ) {
throw new Exception( "手の検出器の作成に失敗しました" );
}
// RealSense カメラであれば、プロパティを設定する
var device = senseManager.QueryCaptureManager().QueryDevice();
PXCMCapture.DeviceInfo dinfo;
device.QueryDeviceInfo( out dinfo );
if ( dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM ) {
device.SetDepthConfidenceThreshold( 1 );
//device.SetMirrorMode( PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED );
device.SetIVCAMFilterOption( 6 );
}
// 手の検出の設定
var config = handAnalyzer.CreateActiveConfiguration();
config.EnableSegmentationImage( true );
config.ApplyChanges();
config.Update();
}
protected virtual bool isDone(PXCMHandData.JointData[] data){
return false;
}
void OnDisable()
{
//Disposses all modules
Initialized = false;
if (SenseManager == null)
{
return;
}
DisposeFunctions.ForEach(i => i.DynamicInvoke());
if (FaceModuleOutput != null)
{
FaceModuleOutput.Dispose();
FaceModuleOutput = null;
}
if (HandDataOutput != null)
{
SenseManager.PauseHand(true);
HandDataOutput.Dispose();
HandDataOutput = null;
}
if (BlobDataOutput != null)
{
SenseManager.PauseBlob(true);
BlobDataOutput.Dispose();
BlobDataOutput = null;
}
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
ImageRgbOutput = null;
}
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
ImageDepthOutput = null;
}
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
ImageIROutput = null;
}
if (Image3DSegmentationOutput != null)
{
Image3DSegmentationOutput.Dispose();
Image3DSegmentationOutput = null;
}
if (Projection != null)
{
Projection.Dispose();
Projection = null;
}
/* GZ
* if (BlobExtractor != null)
* {
* BlobExtractor.Dispose();
* BlobExtractor = null;
* } */
UvMap = null;
PointCloud = null;
SenseManager.Dispose();
SenseManager = null;
}
void OnFiredGesture(PXCMHandData.GestureData data)
{
Debug.Log(data.name.ToString());
string gestureName = data.name.ToString();
if (gestureName.Equals("fist"))
{
gesture = gestureName;
}
else if (gestureName.Equals("v_sign"))
{
gesture = gestureName;
}
else if (gestureName.Equals("thumb_down"))
{
gesture = gestureName;
}
else if (gestureName.Equals("spreadfingers"))
{
gesture = "";
}
}
private static void ProcessGestures(PXCMHandData handData)
{
// Processing gestures
// Querying how many gestures were detected
int firedGesturesNumber = handData.QueryFiredGesturesNumber();
// Querying which gestures were detected
for (int i = 0; i < firedGesturesNumber; i++)
{
PXCMHandData.GestureData gestureData;
if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// Getting the gesture name
String gestureName = gestureData.name;
// Getting the gesture state
PXCMHandData.GestureStateType gestureState = gestureData.state;
// Getting the gesture timestamp
long timestamp = gestureData.timeStamp;
// Getting the hand information about which hand fired the gesture
PXCMHandData.IHand handInfo;
if (handData.QueryHandDataById(gestureData.handId, out handInfo) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// Getting which hand fired the gesture
PXCMHandData.BodySideType bodySideHand = handInfo.QueryBodySide();
if (gestureState == PXCMHandData.GestureStateType.GESTURE_STATE_START)
{
Console.WriteLine("Gesture \"{0}\" was detected. State:{1}, Timestamp:{2}, BodySide:{3}", gestureName, gestureState, timestamp, bodySideHand);
Console.WriteLine("--------------------");
if (gestureName.CompareTo(GESTURE_FIST) == 0)
{
if (currentGesture.CompareTo(GESTURE_FIST) != 0)
{
//SendMqttMessage("surfboard2/relay?0");
String target = (String)iot_form.comboTarget.SelectedItem;
String hclosed = (String)iot_form.comboHandsClosed.SelectedItem;
//if (target.Equals("")) target = "*";
//SendMqttMessage(target + "/" + hclosed);
SendMqttMessage(hclosed);
currentGesture = GESTURE_FIST;
}
}
else if (gestureName.CompareTo(GESTURE_SPREADFINGERS) == 0)
{
if (currentGesture.CompareTo(GESTURE_SPREADFINGERS) != 0)
{
String target = (String)iot_form.comboTarget.SelectedItem;
String hopen = (String)iot_form.comboGive5Action.SelectedItem;
//if (target.Equals("")) target = "*";
SendMqttMessage(hopen);
//SendMqttMessage(target + "/" + hopen);
//System.Diagnostics.Process.Start("http://www.globalcode.com.br");
//SendMqttMessage("surfboard2/relay?1");
currentGesture = GESTURE_SPREADFINGERS;
}
}
else if (gestureName.CompareTo(GESTURE_CLICK) == 0)
{
if (currentGesture.CompareTo(GESTURE_CLICK) != 0)
{
SendMqttMessage("green?255");
currentGesture = GESTURE_CLICK;
}
}
else if (gestureName.CompareTo(GESTURE_VSIGN) == 0)
{
if (currentGesture.CompareTo(GESTURE_VSIGN) != 0)
{
//System.Diagnostics.Process.Start("https://www.youtube.com/watch?v=MVm5hcQYJ-U");
SendMqttMessage("green?0");
keepLooping = false;
currentGesture = GESTURE_VSIGN;
}
}
}
}
}
}