public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is EventTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
//AcquireFrame
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0)
{
int totalNumOfFiredGestures = SenseToolkitManager.Instance.HandDataOutput.QueryFiredGesturesNumber();
PXCMHandData.GestureData gestureData;
for (int i = 0; i < totalNumOfFiredGestures; i++)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryFiredGestureData(i, out gestureData) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMHandData.IHand handData = null;
if ((ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out handData) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out handData) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
_uniqueID = handData.QueryUniqueId();
if (handData.QueryUniqueId() == gestureData.handId)
{
MCTTypes.RSUnityToolkitGestures firedGesture = MCTTypes.GetGesture(gestureData.name);
if (((!Gesture.Equals(MCTTypes.RSUnityToolkitGestures.None)) && Gesture.Equals(firedGesture)))
{
if (!_lastFrameDetected)
{
_lastFrameDetected = true;
return(true);
}
else
{
return(false);
}
}
}
}
}
}
_lastFrameDetected = false;
}
}
return(false);
}
private void SetHandPosition(Hand hand, PXCMHandData.IHand handInfo)
{
var imagePosition = ToPoint3D(handInfo.QueryMassCenterImage());
var worldPosition = ToPoint3D(handInfo.QueryMassCenterWorld());
hand.Position = CreatePosition(imagePosition, worldPosition);
}
private void RecogMove(PXCMHandData.IHand hand, PXCMPointF32 mcp)
{
int side = (int)hand.QueryBodySide() - 1;
if (side < 0)
{
return;
}
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out middleData[side]);
if (oldMiddleData[side] == null)
{
oldMiddleData[side] = middleData[side];
return;
}
//Console.WriteLine(middleData[side].speed.x);
var distance = Math.Pow(
Math.Pow(middleData[side].positionWorld.x - oldMiddleData[side].positionWorld.x, 2)
+ Math.Pow(middleData[side].positionWorld.y - oldMiddleData[side].positionWorld.y, 2)
+ Math.Pow((middleData[side].positionWorld.z - oldMiddleData[side].positionWorld.z) * 1000, 2),
0.5);
oldMiddleData[side] = middleData[side];
//Console.Write(middleData[side].positionImage.x + ":" + oldMiddleData[side].positionImage.x + ":");
//Console.WriteLine(middleData[side].positionImage.x - oldMiddleData[side].positionImage.x);
//oldMiddleData = middleData;
}
public void updateData(PXCMHandData.IHand handData)
{
this.handJoint = new PXCMHandData.JointData[22];
var jointTypes = Enum.GetValues(typeof(PXCMHandData.JointType)).Cast <PXCMHandData.JointType>();
int index = 0;
foreach (var jtmp in jointTypes)
{
PXCMHandData.JointData jdata = null;
handData.QueryTrackedJoint(jtmp, out jdata);
handJoint[index] = jdata;
index++;
}
index = 0;
this.handFinger = new PXCMHandData.FingerData[5];
var fingerTypes = Enum.GetValues(typeof(PXCMHandData.FingerType)).Cast <PXCMHandData.FingerType>();
foreach (var fmp in fingerTypes)
{
PXCMHandData.FingerData fdata = null;
handData.QueryFingerData(fmp, out fdata);
handFinger[index] = fdata;
index++;
}
this.handContourInner = new List <PXCMPointI32[]>();
this.handContourOuter = new List <PXCMPointI32[]>();
int contourNumber = handData.QueryNumberOfContours();
if (contourNumber > 0)
{
PXCMPointI32[] pointOuter;
PXCMPointI32[] pointInner;
for (int k = 0; k < contourNumber; ++k)
{
PXCMHandData.IContour contour;
pxcmStatus sts = handData.QueryContour(k, out contour);
if (sts == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (contour.IsOuter() == true)
{
contour.QueryPoints(out pointOuter);
handContourOuter.Add(pointOuter);
}
else
{
contour.QueryPoints(out pointInner);
handContourInner.Add(pointInner);
}
}
}
}
}
private void TrackPinky(Finger finger, PXCMHandData.IHand handInfo)
{
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_PINKY_BASE, finger.BaseJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_PINKY_JT1, finger.FirstJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_PINKY_JT2, finger.SecondJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_PINKY_TIP, finger);
SetFingerOpenness(finger, PXCMHandData.FingerType.FINGER_PINKY, handInfo);
}
private void TrackThumb(Finger finger, PXCMHandData.IHand handInfo)
{
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_THUMB_BASE, finger.BaseJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_THUMB_JT1, finger.FirstJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_THUMB_JT2, finger.SecondJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_THUMB_TIP, finger);
SetFingerOpenness(finger, PXCMHandData.FingerType.FINGER_THUMB, handInfo);
}
private void TrackIndex(Finger finger, PXCMHandData.IHand handInfo)
{
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_BASE, finger.BaseJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_JT1, finger.FirstJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_JT2, finger.SecondJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_INDEX_TIP, finger);
SetFingerOpenness(finger, PXCMHandData.FingerType.FINGER_INDEX, handInfo);
}
private void TrackMiddle(Finger finger, PXCMHandData.IHand handInfo)
{
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_MIDDLE_BASE, finger.BaseJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_MIDDLE_JT1, finger.FirstJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_MIDDLE_JT2, finger.SecondJoint);
SetJointdata(handInfo, PXCMHandData.JointType.JOINT_MIDDLE_TIP, finger);
SetFingerOpenness(finger, PXCMHandData.FingerType.FINGER_MIDDLE, handInfo);
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is EventTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
EventTrigger specificTrigger = (EventTrigger)trigger;
specificTrigger.Source = this.name;
bool success = false;
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
PXCMHandData.IHand data = null;
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0
&&
(
(ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
)
{
_uniqueID = data.QueryUniqueId();
_lastFrameDetected = true;
}
else
{
if (_lastFrameDetected)
{
success = true;
}
_lastFrameDetected = false;
}
}
else
{
return(false);
}
return(success);
}
private void SetJointdata(PXCMHandData.IHand handInfo, PXCMHandData.JointType jointType, Item joint)
{
PXCMHandData.JointData jointData;
if (handInfo.QueryTrackedJoint(jointType, out jointData) != NoError)
{
joint.IsVisible = false;
return;
}
SetVisibleJointPosition(joint, jointData);
}
private Hand GetHandData(PXCMHandData.IHand currentHandData)
{
PXCMHandData.JointData jointData;
currentHandData.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out jointData);
var positionImage = jointData.positionWorld;
float z = positionImage.z;
var center = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_CENTER);
Hand hand = new Hand
{
Thumb = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_BASE),
Center = center
},
Index = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_BASE),
Center = center
},
Middle = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_BASE),
Center = center
},
Ring = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_BASE),
Center = center
},
Pinky = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_BASE),
Center = center
},
z = z
};
return(hand);
}
private void SetHandPosition(Hand hand, PXCMHandData.IHand handInfo)
{
var world = handInfo.QueryMassCenterWorld();
if (_smoothers.ContainsKey(hand))
{
world = _smoothers[hand].SmoothValue(world);
}
var imagePosition = ToPoint3D(handInfo.QueryMassCenterImage());
var worldPosition = ToPoint3D(world);
hand.Position = CreatePosition(imagePosition, worldPosition);
}
private static Point GetPointFromJoint(PXCMHandData.IHand currentHandData, PXCMHandData.JointType jointType)
{
PXCMHandData.JointData jointData;
currentHandData.QueryTrackedJoint(jointType, out jointData);
var positionImage = jointData.positionImage;
int x = (int)(WIDTH - positionImage.x);
int y = (int)positionImage.y;
return(new Point
{
X = x,
Y = y
});
}
public override void Work(Graphics g)
{
data = module.CreateOutput();
data.Update();
PXCMHandData.IHand hands = null;
data.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, 0, out hands);
if (hands == null)
{
return;
}
PXCMRectI32 rect = hands.QueryBoundingBoxImage();
Rectangle rectangle = new Rectangle(rect.x, rect.y, rect.w, rect.h); // Convert to Rectangle
g.DrawRectangle(pen, rectangle); // Draw
}
private void SetHandOrientation(Hand hand, PXCMHandData.IHand handInfo)
{
var d4 = handInfo.QueryPalmOrientation();
PXCMRotation rotationHelper;
_camera.Session.CreateImpl(out rotationHelper);
rotationHelper.SetFromQuaternion(d4);
var rotationEuler = rotationHelper.QueryEulerAngles(PXCMRotation.EulerOrder.PITCH_YAW_ROLL);
var x = rotationEuler.x * 180 / Math.PI;
var y = rotationEuler.y * 180 / Math.PI;
var z = rotationEuler.z * 180 / Math.PI;
hand.Rotation = new Rotation(x, y, z);
rotationHelper.Dispose();
}
public override bool Process(Trigger trigger)
{
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is EventTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 1)
{
PXCMHandData.IHand leftHand = null;
PXCMHandData.IHand rightHand = null;
//Query both hands
if (SenseToolkitManager.Instance.HandDataOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out leftHand) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out rightHand) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (!_twoHandsDetected)
{
_twoHandsDetected = true;
}
return(false);
}
}
}
}
if (_twoHandsDetected)
{
_twoHandsDetected = false;
return(true);
}
return(false);
}
/// <summary> 指のデータを取得する </summary>
private void GetFingerData(PXCMHandData.IHand hand, PXCMHandData.JointType jointType)
{
PXCMHandData.JointData jointData;
var sts = hand.QueryTrackedJoint(jointType, out jointData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
// Depth座標系をカラー座標系に変換する
var depthPoint = new PXCMPoint3DF32[1];
var colorPoint = new PXCMPointF32[1];
depthPoint[0].x = jointData.positionImage.x;
depthPoint[0].y = jointData.positionImage.y;
depthPoint[0].z = jointData.positionWorld.z * 1000;
projection.MapDepthToColor(depthPoint, colorPoint);
var masp = hand.QueryMassCenterImage();
var mdp = new PXCMPoint3DF32[1];
var mcp = new PXCMPointF32[1];
mdp[0].x = masp.x;
mdp[0].y = masp.y;
mdp[0].z = hand.QueryMassCenterWorld().z * 1000;
projection.MapDepthToColor(mdp, mcp);
//Console.WriteLine(mcp[0].x);
AddEllipse(new Point(mcp[0].x, mcp[0].y), 10, Brushes.Red, 1);
colorPoint = mcp;
//AddEllipse(new Point(colorPoint[0].x, colorPoint[0].y), 5, Brushes.White, 1);
if (ensembleTimer.IsEnabled)
{
DetectTap(hand, mcp[0]);
}
//if (ensembleTimer.IsEnabled) RecogMove(hand,mcp[0]);
//RecogMove(hand,mcp[0]);
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is TrackTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
// make sure we have valid values
if (RealWorldBoxDimensions.x <= 0)
{
RealWorldBoxDimensions.x = 1;
}
if (RealWorldBoxDimensions.y <= 0)
{
RealWorldBoxDimensions.y = 1;
}
if (RealWorldBoxDimensions.z <= 0)
{
RealWorldBoxDimensions.z = 1;
}
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
PXCMHandData.IHand data = null;
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0 &&
((ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
)
{
// Process Tracking
_uniqueID = data.QueryUniqueId();
PXCMHandData.JointData jointData;
data.QueryTrackedJoint(TrackedJoint, out jointData);
TrackTrigger specificTrigger = (TrackTrigger)trigger;
PXCMPoint3DF32 point = jointData.positionWorld;
Vector3 position = new Vector3(point.x, point.y, point.z);
position.x *= -100;
position.y *= 100;
position.z *= 100;
if (position.x + position.y + position.z == 0)
{
return(false);
}
TrackingUtilityClass.ClampToRealWorldInputBox(ref position, RealWorldBoxCenter, RealWorldBoxDimensions);
TrackingUtilityClass.Normalize(ref position, RealWorldBoxCenter, RealWorldBoxDimensions);
if (!float.IsNaN(position.x) && !float.IsNaN(position.y) && !float.IsNaN(position.z))
{
specificTrigger.Position = position;
}
else
{
return(false);
}
Quaternion q = new Quaternion(jointData.globalOrientation.x, jointData.globalOrientation.y,
jointData.globalOrientation.z, jointData.globalOrientation.w);
q = q * _zInvert * _yInvert;
specificTrigger.RotationQuaternion = q;
success = true;
}
}
else
{
return(false);
}
return(success);
}
pxcmStatus newHandFrame(PXCMHandModule hand)
{
if (hand != null)
{
PXCMHandData handData = hand.CreateOutput();
handData.Update();
PXCMHandData.IHand iHandDataLeft = null, iHandDataRight = null;
PXCMHandData.JointData jointData = null;
PXCMImage image = null;
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out iHandDataLeft);
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out iHandDataRight);
if (handForm != null && !handForm.IsDisposed)
{
this.handForm.HandCount = handData.QueryNumberOfHands();
if (iHandDataLeft != null)
{
iHandDataLeft.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.LeftHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
if (iHandDataRight != null)
{
iHandDataRight.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.RightHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
}
if (iHandDataLeft != null)
{
if (jointData == null)
{
iHandDataLeft.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (iHandDataRight != null)
{
if (jointData == null)
{
iHandDataRight.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (jointData != null && canTrack.Checked)
{
Cursor.Position = new System.Drawing.Point(
(int)((640.0f - jointData.positionImage.x) * Screen.PrimaryScreen.Bounds.Width / 640.0f),
(int)(jointData.positionImage.y * Screen.PrimaryScreen.Bounds.Height / 480.0f));
PXCMHandData.GestureData gestureData = null;
if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
Program.DoMouseClick();
}
Console.WriteLine("Z Position: " + jointData.positionWorld.z);
}
handData.Dispose();
}
return(pxcmStatus.PXCM_STATUS_NO_ERROR);
}
private void DetectTap(PXCMHandData.IHand hand)
{
PXCMHandData.JointData MiddleData;
PXCMHandData.JointData CenterData;
//指のデータをとってくる(depth)
//ユーザの右手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out CenterData);
RightCenter = CenterData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out MiddleData);
RightMiddle = MiddleData.positionWorld;
//RightCenter = hand.QueryMassCenterWorld();
}
//ユーザの左手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out CenterData);
LeftCenter = CenterData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out MiddleData);
LeftMiddle = MiddleData.positionWorld;
}
//if文の条件を記述(前の指のデータと比較)
// ユーザの右手でタップ
if (-RightMiddle.z + preRightMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
System.Math.Pow(System.Math.Pow(RightMiddle.x - preRightMiddle.x, 2) // 指先の速度が1.8m/s以上
+ System.Math.Pow(RightMiddle.y - preRightMiddle.y, 2)
+ System.Math.Pow(RightMiddle.z * 1000 - preRightMiddle.z * 1000, 2), 0.5) > 0.03 &&
System.Math.Pow(System.Math.Pow(RightCenter.x - preRightCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ System.Math.Pow(RightCenter.y - preRightCenter.y, 2)
+ System.Math.Pow(RightCenter.z * 1000 - preRightCenter.z * 1000, 2), 0.5) > 0.01 &&
rsw.ElapsedMilliseconds > 250
)
{
//tap音を出力
midiManager.SetOnNote(player.MusicTime);
rsw.Restart();
}
// ユーザの左手でタップ
if (-LeftMiddle.z + preLeftMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
System.Math.Pow(System.Math.Pow(LeftMiddle.x - preLeftMiddle.x, 2) // 指先の速度が1.8m/s以上
+ System.Math.Pow(RightMiddle.y - preLeftMiddle.y, 2)
+ System.Math.Pow(RightMiddle.z - preLeftMiddle.z, 2), 0.5) > 0.03 &&
System.Math.Pow(System.Math.Pow(RightCenter.x - preLeftCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ System.Math.Pow(RightCenter.y - preLeftCenter.y, 2)
+ System.Math.Pow(RightCenter.z - preLeftCenter.z, 2), 0.5) > 0.01 &&
System.Math.Pow(System.Math.Pow(RightCenter.x - preLeftCenter.x, 2) // 手のひらの速度が1.5m/s以下
+ System.Math.Pow(RightCenter.y - preLeftCenter.y, 2)
+ System.Math.Pow(RightCenter.z - preLeftCenter.z, 2), 0.5) < 0.025
)
{
//tap音を出力
midiManager.SetOnNote(player.MusicTime);
}
//Console.WriteLine("RightCenter.x:" + RightCenter.x);
//Console.WriteLine("preRightCenter.x:" + preRightCenter.x);
//Console.WriteLine();
//Console.WriteLine("RightMiddle.x:" + RightMiddle.x);
//Console.WriteLine("preRightMiddle.x:" + preRightMiddle.x);
//前の指のデータに今の指のデータを上書き
//plc,preLeftMiddle,preRightCenter,preRightMiddle
// 上手くいかなければディープコピーする．
preRightCenter.x = RightCenter.x;
preRightCenter.y = RightCenter.y;
preRightCenter.z = RightCenter.z;
preRightMiddle.x = RightMiddle.x;
preRightMiddle.y = RightMiddle.y;
preRightMiddle.z = RightMiddle.z;
preLeftCenter.x = LeftCenter.x;
preLeftCenter.y = LeftCenter.y;
preLeftCenter.z = LeftCenter.z;
preLeftMiddle.x = LeftMiddle.x;
preLeftMiddle.y = LeftMiddle.y;
preLeftMiddle.z = LeftMiddle.z;
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is EventTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
//AcquireFrame
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
PXCMHandData.IHand data = null;
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0
&&
(
(ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
)
{
_uniqueID = data.QueryUniqueId();
PXCMHandData.JointData jointD;
data.QueryTrackedJoint(PXCMHandData.JointType.JOINT_PINKY_JT2, out jointD);
if (jointD.confidence < 100)
{
return(false);
}
if (data.QueryOpenness() <= OpennessFactor)
{
_lastFrameDetected = true;
}
else
{
if (_lastFrameDetected)
{
success = true;
}
_lastFrameDetected = false;
}
}
else
{
if (_lastFrameDetected)
{
success = true;
}
_lastFrameDetected = false;
}
}
return(success);
}
private void SetHandOpenness(Hand hand, PXCMHandData.IHand handInfo)
{
int openness = handInfo.QueryOpenness();
SetOpenness(hand, openness);
}
private void SetFingerOpenness(Finger finger, PXCMHandData.FingerType fingerType, PXCMHandData.IHand handInfo)
{
PXCMHandData.FingerData fingerData;
if (handInfo.QueryFingerData(fingerType, out fingerData) != NoError)
{
return;
}
SetOpenness(finger, fingerData.foldedness);
}
private void DetectTap(PXCMHandData.IHand hand, PXCMPointF32 mcp)
{
PXCMHandData.JointData midData;
PXCMHandData.JointData cntData;
//指のデータをとってくる(depth)
//ユーザの右手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out cntData);
RightCenter = cntData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out midData);
RightMiddle = midData.positionWorld;
}
//ユーザの左手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out cntData);
LeftCenter = cntData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out midData);
LeftMiddle = midData.positionWorld;
}
if (mcp.y < 0)
{
mcp.y = 0;
}
if (mcp.y > ColorImage.Height)
{
mcp.y = (float)ColorImage.Height;
}
//Console.WriteLine(rsw.ElapsedMilliseconds);
//if文の条件を記述(前の指のデータと比較)
// ユーザの右手でタップ
if (-RightMiddle.z + preRightMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
Math.Pow(Math.Pow(RightMiddle.x - preRightMiddle.x, 2) // 指先の速度が1.8m/s以上
+ Math.Pow(RightMiddle.y - preRightMiddle.y, 2)
+ Math.Pow(RightMiddle.z * 1000 - preRightMiddle.z * 1000, 2), 0.5) > 0.03 &&
Math.Pow(Math.Pow(RightCenter.x - preRightCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ Math.Pow(RightCenter.y - preRightCenter.y, 2)
+ Math.Pow(RightCenter.z * 1000 - preRightCenter.z * 1000, 2), 0.5) > 0.01 &&
rsw.ElapsedMilliseconds > 300
)
{
//tap音を出力
midi.OnNote(currentFreqs[(int)(((ColorImage.Height - mcp.y) / (ColorImage.Height + 0.01)) * currentFreqs.Length)]);
rsw.Restart();
}
//var c = ColorImage.Height - y;
//var e = c / ColorImage.Height;
//var d = e * currentFreqs.Length;
//Index.Text = y + "\n" + d + "\n" + (int)d;
//Index.Text = mcp.y.ToString();
//Chord.Text = e.ToString();
// ユーザの左手でタップ
if (-LeftMiddle.z + preLeftMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
Math.Pow(Math.Pow(LeftMiddle.x - preLeftMiddle.x, 2) // 指先の速度が1.8m/s以上
+ Math.Pow(LeftMiddle.y - preLeftMiddle.y, 2)
+ Math.Pow(LeftMiddle.z * 1000 - preLeftMiddle.z * 1000, 2), 0.5) > 0.03 &&
Math.Pow(Math.Pow(LeftCenter.x - preLeftCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ Math.Pow(LeftCenter.y - preLeftCenter.y, 2)
+ Math.Pow(LeftCenter.z * 1000 - preLeftCenter.z * 1000, 2), 0.5) > 0.01 &&
lsw.ElapsedMilliseconds > 300
)
{
//tap音を出力
//midi.OnNote(currentFreqs[(int)(((ColorImage.Height - mcp.y) / (ColorImage.Height + 0.01)) * currentFreqs.Length)]);
lsw.Restart();
}
//Console.WriteLine("RightCenter.x:" + RightCenter.x);
//Console.WriteLine("preRightCenter.x:" + preRightCenter.x);
//Console.WriteLine();
//Console.WriteLine("RightMiddle.x:" + RightMiddle.x);
//Console.WriteLine("preRightMiddle.x:" + preRightMiddle.x);
//前の指のデータに今の指のデータを上書き
//plc,preLeftMiddle,preRightCenter,preRightMiddle
// 上手くいかなければディープコピーする．
preRightCenter.x = RightCenter.x;
preRightCenter.y = RightCenter.y;
preRightCenter.z = RightCenter.z;
preRightMiddle.x = RightMiddle.x;
preRightMiddle.y = RightMiddle.y;
preRightMiddle.z = RightMiddle.z;
preLeftCenter.x = LeftCenter.x;
preLeftCenter.y = LeftCenter.y;
preLeftCenter.z = LeftCenter.z;
preLeftMiddle.x = LeftMiddle.x;
preLeftMiddle.y = LeftMiddle.y;
preLeftMiddle.z = LeftMiddle.z;
}
/// <summary> 指のデータを取得する </summary>
private bool GetFingerData(PXCMHandData.IHand hand, PXCMHandData.JointType jointType)
{
PXCMHandData.JointData jointData;
var sts = hand.QueryTrackedJoint(jointType, out jointData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return(false);
}
// Depth座標系をカラー座標系に変換する
var depthPoint = new PXCMPoint3DF32[1];
var colorPoint = new PXCMPointF32[1];
depthPoint[0].x = jointData.positionImage.x;
depthPoint[0].y = jointData.positionImage.y;
depthPoint[0].z = jointData.positionWorld.z * 1000;
projection.MapDepthToColor(depthPoint, colorPoint);
var masp = hand.QueryMassCenterImage();
var mdp = new PXCMPoint3DF32[1];
var mcp = new PXCMPointF32[1];
mdp[0].x = masp.x;
mdp[0].y = masp.y;
mdp[0].z = hand.QueryMassCenterWorld().z * 1000;
projection.MapDepthToColor(mdp, mcp);
//Console.WriteLine(mcp[0].x);
AddEllipse(new Point(mcp[0].x, mcp[0].y), 10, Brushes.Red, 1);
colorPoint = mcp;
//ユーザの右手に対して演奏領域の当たり判定確認
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
for (int i = 0; i < 5; i++)
{
if ((imageColor.Height / 5) * i <= colorPoint[0].y && colorPoint[0].y < (imageColor.Height / 5) * (i + 1))
{
if (16 - i != NowRange)
{
NowRange = 16 - i;
PivotList.Dispatcher.BeginInvoke(
new Action(() =>
{
PivotList.SelectedItem = NowRange;
}
));
}
}
}
}
//ユーザの左手に対してアイコンの当たり判定の確認
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
IconHitCheck(colorPoint[0]);
}
AddEllipse(new Point(colorPoint[0].x, colorPoint[0].y), 5, Brushes.White, 1);
return(true);
}
// Update is called once per frame
void Update () {
/* Make sure SenseManager Instance is valid */
if (senseManager == null)
return;
/* Wait until any frame data is available */
if (senseManager.AcquireFrame (false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
return;
/* Retrieve hand tracking Module Instance */
handAnalyzer = senseManager.QueryHand ();
try
{
if (handAnalyzer != null) {
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput ();
if (_handData != null) {
_handData.Update ();
PXCMHandData.IHand[] _iHand = new PXCMHandData.IHand[MaxHands];
NumOfHands = _handData.QueryNumberOfHands ();
hands.isLeft = false;
hands.isRight = false;
/* Retrieve all joint Data */
if (_handData.QueryHandData (PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out _iHand[0]) == pxcmStatus.PXCM_STATUS_NO_ERROR) {
/*Identify left/right hand */
gesture[0].isExist = true;
hands.isLeft = true;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++){
//Debug.Log (i.ToString());
if (_handData.QueryFiredGestureData (i, out gesture[0].gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.Log (gesture[0].gestureData.name);
//continue;//If you want to use this gesture info, you need to realize a SEND function here.
}
for (int j = 0; j < MaxJoints; j++) {
if (_iHand[0].QueryTrackedJoint ((PXCMHandData.JointType)j, out hands.jointData [0] [j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
hands.jointData [0] [j] = null;
/* hands.smoothPosition[0][j].AddSample(hands.jointData[0][j].positionWorld);
hands.smoothLocalRotation[0][j].AddSample(((Quaternion)hands.jointData[0][j].localRotation).eulerAngles);
hands.smoothGlobalRotation[0][j].AddSample(((Quaternion)hands.jointData[0][j].globalOrientation).eulerAngles);
*/}
if (!handList.ContainsKey (_iHand[0].QueryUniqueId ()))
handList.Add (_iHand[0].QueryUniqueId (), _iHand[0].QueryBodySide ());
}else{
gesture[0].isExist = false;
}
if (_handData.QueryHandData (PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out _iHand[1]) == pxcmStatus.PXCM_STATUS_NO_ERROR) {
/*Identify left/right hand */
gesture[1].isExist = true;
hands.isRight = true;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++)
if (_handData.QueryFiredGestureData (i, out gesture[1].gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
continue;//If you want to use this gesture info, you need to realize a SEND function here.
for (int j = 0; j < MaxJoints; j++) {
if (_iHand[1].QueryTrackedJoint ((PXCMHandData.JointType)j, out hands.jointData [1] [j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
hands.jointData [1] [j] = null;
}
if (!handList.ContainsKey (_iHand[1].QueryUniqueId ()))
handList.Add (_iHand[1].QueryUniqueId (), _iHand[1].QueryBodySide ());
}else{
gesture[1].isExist = false;
}
}
_handData.Dispose ();
}
}
catch (IOException ex)
{
Console.WriteLine("An IOException has been thrown!");
Console.WriteLine(ex.ToString());
Console.ReadLine();
return;
}
handAnalyzer.Dispose ();
senseManager.ReleaseFrame ();
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
Debug.LogError("Hand Analysis Module Not Set");
trigger.ErrorDetected = true;
return(false);
}
if (!(trigger is TranslationTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
// make sure we have valid values
if (RealWorldBoxDimensions.x <= 0)
{
RealWorldBoxDimensions.x = 1;
}
if (RealWorldBoxDimensions.y <= 0)
{
RealWorldBoxDimensions.y = 1;
}
if (RealWorldBoxDimensions.z <= 0)
{
RealWorldBoxDimensions.z = 1;
}
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
PXCMHandData.IHand data = null;
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0
&&
(
(ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
)
{
_uniqueID = data.QueryUniqueId();
// Process Translation
if (trigger is TranslationTrigger)
{
TranslationTrigger specificTrigger = (TranslationTrigger)trigger;
PXCMHandData.JointData jointData;
data.QueryTrackedJoint(TrackedJoint, out jointData);
Vector3 vecPos = new Vector3(-(jointData.positionWorld.x), (jointData.positionWorld.y), jointData.positionWorld.z);
vecPos.x *= 100;
vecPos.y *= 100;
vecPos.z *= 100;
TrackingUtilityClass.ClampToRealWorldInputBox(ref vecPos, RealWorldBoxCenter, RealWorldBoxDimensions);
TrackingUtilityClass.Normalize(ref vecPos, RealWorldBoxCenter, RealWorldBoxDimensions);
if (specificTrigger.Restart)
{
specificTrigger.Restart = false;
_referencePosition = vecPos;
return(false);
}
specificTrigger.Translation = vecPos - _referencePosition;
Vector3 passCriteria = new Vector3();
passCriteria.x = Mathf.Abs(vecPos.x) - Mathf.Abs(Thresholds.x);
passCriteria.y = Mathf.Abs(vecPos.y) - Mathf.Abs(Thresholds.y);
passCriteria.z = Mathf.Abs(vecPos.z) - Mathf.Abs(Thresholds.z);
if (passCriteria.x > 0 || passCriteria.y > 0 || passCriteria.z > 0)
{
_referencePosition = vecPos;
success = true;
}
}
}
}
else
{
return(false);
}
return(success);
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is RotationTrigger || trigger is ScaleTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
#region Rotation
//Rotation
if (trigger is RotationTrigger)
{
//AcquireFrame
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 1)
{
PXCMHandData.IHand leftHand = null;
PXCMHandData.IHand rightHand = null;
//Query both hands
if (SenseToolkitManager.Instance.HandDataOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out leftHand) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out rightHand) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
RotationTrigger rotationTrig = (RotationTrigger)trigger;
float yCurDiff = rightHand.QueryMassCenterWorld().y * 100 - leftHand.QueryMassCenterWorld().y * 100;
float zCurDiff = rightHand.QueryMassCenterWorld().z * 100 - leftHand.QueryMassCenterWorld().z * 100;
//Initialization
if (rotationTrig.Restart)
{
_roll = yCurDiff;
_pitch = (float)(Math.Atan2(yCurDiff, zCurDiff));
_yaw = zCurDiff;
rotationTrig.Restart = false;
}
rotationTrig.Roll = yCurDiff - _roll;
rotationTrig.Roll *= -1;
if (Math.Abs(rotationTrig.Roll) < DistanceThresholdX)
{
rotationTrig.Roll = 0;
}
rotationTrig.Pitch = (float)(Math.Atan2(yCurDiff, zCurDiff) - _pitch);
if (Math.Abs(rotationTrig.Pitch) < DistanceThresholdZ)
{
rotationTrig.Pitch = 0;
}
rotationTrig.Yaw = zCurDiff - _yaw;
if (Math.Abs(rotationTrig.Yaw) < DistanceThresholdY)
{
rotationTrig.Yaw = 0;
}
if ((rotationTrig.Roll == rotationTrig.Yaw) && (rotationTrig.Yaw == rotationTrig.Pitch) && (rotationTrig.Pitch == 0))
{
return(false);
}
return(true);
}
}
}
}
return(false);
}
#endregion
#region Scale
//Scale
if (trigger is ScaleTrigger)
{
//AcquireFrame
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 1)
{
PXCMHandData.IHand leftHand = null;
PXCMHandData.IHand rightHand = null;
//Query both hands
if (SenseToolkitManager.Instance.HandDataOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out leftHand) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (SenseToolkitManager.Instance.HandDataOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out rightHand) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
ScaleTrigger scaleTrig = (ScaleTrigger)trigger;
float xCurDiff = rightHand.QueryMassCenterWorld().x * 100 - leftHand.QueryMassCenterWorld().x * 100;
//Initialization
if (scaleTrig.Restart)
{
_scale = xCurDiff;
scaleTrig.Restart = false;
scaleTrig.Scale = 0;
}
else
{
scaleTrig.Scale = _scale - xCurDiff;
}
if (Math.Abs(scaleTrig.Scale) < DistanceThresholdX)
{
scaleTrig.Scale = 0;
return(false);
}
return(true);
}
}
}
}
return(false);
}
#endregion
return(false);
}
/// <summary>
/// Acquire the frames from streams
/// </summary>
private void AcquireFrames()
{
Console.WriteLine("RealSense THREAD ID: " + System.Threading.Thread.CurrentThread.ManagedThreadId);
PXCMImage.ImageData segmented_image_data;
PXCMImage segmentedImage;
PXCM3DSeg pSeg;
PXCMCapture.Sample sample;
PXCMImage.ImageData colorData;
PXCMImage.ImageData depthData;
//PXCMEmotion pEmo;
while (manager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (streamCancellationToken.IsCancellationRequested)
{
break;
}
sample = manager.QuerySample();
//Color analysis
if (isColorStreamEnabled && sample.color != null)
{
if (sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
parentContext.Post((object state) =>
{
if (ColorFrameReady != null) ColorFrameReady(this, new RealSenseEventArgs(colorData, sample.color.info));
}, "");
}
else
{
parentContext.Post((object state) =>
{
if (ColorFrameReady != null) ColorFrameReady(this, new RealSenseEventArgs());
}, "");
}
sample.color.ReleaseAccess(colorData);
sample.color.Dispose();
}
//Depth analysis
if (isDepthStreamEnabled && sample.depth != null)
{
if (sample.depth.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out depthData) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
parentContext.Post((object state) =>
{
if (DepthFrameReady != null) DepthFrameReady(this, new RealSenseEventArgs(depthData, sample.depth.info));
}, "");
}
else
{
parentContext.Post((object state) =>
{
if (DepthFrameReady != null) DepthFrameReady(this, new RealSenseEventArgs());
}, "");
}
sample.depth.ReleaseAccess(depthData);
sample.depth.Dispose();
}
//Gesture analysis
if (isGestureRecognitionEnabled)
{
this.outputData.Update();
int numberOfHands = this.outputData.QueryNumberOfHands();
PXCMHandData.IHand handData;
PXCMHandData.GestureData gestureData;
for (int i = 0; i < outputData.QueryFiredGesturesNumber(); i++)
{
try
{
if (outputData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
outputData.QueryHandDataById(gestureData.handId, out handData);
var bodySide = handData.QueryBodySide();
//Raise event from the parent context, not from the new started thread
parentContext.Post((object state) =>
{
if (GestureRecognized != null) GestureRecognized(this, new RealSenseGestureEventArgs(gestureData, bodySide));
}, "");
}
}
catch (Exception e) { }
}
}
//Hand analysis
if (isHandsStreamEnabled)
{
this.outputData.Update();
Dictionary<PXCMHandData.JointType, PXCMHandData.JointData>[] hands = new Dictionary<PXCMHandData.JointType, PXCMHandData.JointData>[4];
PXCMHandData.IHand[] totalHands = new PXCMHandData.IHand[4];
int numberOfHands = outputData.QueryNumberOfHands();
if (numberOfHands > 0)
{
for (int i = 0; i < numberOfHands; i++)
{
Dictionary<PXCMHandData.JointType, PXCMHandData.JointData> handDictionary = new Dictionary<PXCMHandData.JointType, PXCMHandData.JointData>();
PXCMHandData.JointData temp;
PXCMHandData.IHand handData;
if (outputData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out handData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// iterate through Joints
for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++)
{
handData.QueryTrackedJoint((PXCMHandData.JointType)j, out temp);
handDictionary.Add((PXCMHandData.JointType)j, temp);
}
hands[i] = handDictionary;
totalHands[i] = handData;
}
}
//Raise event from the parent context, not from the new started thread
parentContext.Post((object state) =>
{
if (HandsFrameReady != null) HandsFrameReady(this, new RealSenseHandsEventArgs(hands, totalHands));
}, "");
}
}
if (is3DSegEnabled)
{
pSeg = manager.Query3DSeg();
segmentedImage = pSeg.AcquireSegmentedImage();
if (segmentedImage != null)
{
segmentedImage.AcquireAccess(
PXCMImage.Access.ACCESS_READ_WRITE,
PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32,
out segmented_image_data);
int height = segmentedImage.QueryInfo().height;
int width = segmentedImage.QueryInfo().width;
for (int y = 0; y < height; y++)
{
unsafe
{
byte* p = (byte*)segmented_image_data.planes[0] + y * segmented_image_data.pitches[0];
const byte grey = 0x7f;
for (int x = 0; x < width; x++)
{
if (p[3] > 0)
{
// When the user moves into the near/far extent, the alpha values will drop from 255 to 1.
// This can be used to fade the user in/out as a cue to move into the ideal operating range.
float blend_factor = 1;//(float)p[3] / (float)255;
for (int ch = 0; ch < 3; ch++)
{
byte not_visible = (byte)((p[ch] >> 4) + grey);
p[ch] = (byte)(p[ch] * blend_factor + not_visible * (1.0f - blend_factor));
}
}
else
{
for (int ch = 0; ch < 3; ch++) p[ch] = (byte)(0xFF); // p[ch] = (byte)((p[ch] >> 4) + grey);
}
p += 4;
}
}
}
parentContext.Post((object state) =>
{
if (SegmentedFrameReady != null) SegmentedFrameReady(this, new RealSenseEventArgs(segmented_image_data, segmentedImage.info));
}, "");
// release the image
segmentedImage.ReleaseAccess(segmented_image_data);
segmentedImage.Dispose();
}
}
//if (isEmotionEnabled)
//{
// pEmo = manager.QueryEmotion();
// int numFaces = pEmo.QueryNumFaces();
// for (int i = 0; i < numFaces; i++)
// {
// /* Retrieve emotionDet location data */
// PXCMEmotion.EmotionData[] data = new PXCMEmotion.EmotionData[10];
// if (pEmo.QueryAllEmotionData(i, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
// {
// bool emotionPresent = false;
// int epidx = -1;
// int maxscoreE = -3;
// float maxscoreI = 0;
// for (int e = 0; e < NUM_EMOTIONS; e++)
// {
// if (data[e].evidence < maxscoreE) continue;
// if (data[e].intensity < maxscoreI) continue;
// maxscoreE = data[e].evidence;
// maxscoreI = data[e].intensity;
// epidx = e;
// }
// if ((epidx != -1) && (maxscoreI > 0.4))
// {
// emotionPresent = true;
// }
// int spidx = -1;
// if (emotionPresent)
// {
// maxscoreE = -3;
// maxscoreI = 0;
// for (int e = 0; e < (NUM_EMOTIONS - NUM_PRIMARY_EMOTIONS); e++)
// {
// if (data[NUM_PRIMARY_EMOTIONS + e].evidence < maxscoreE) continue;
// if (data[NUM_PRIMARY_EMOTIONS + e].intensity < maxscoreI) continue;
// maxscoreE = data[NUM_PRIMARY_EMOTIONS + e].evidence;
// maxscoreI = data[NUM_PRIMARY_EMOTIONS + e].intensity;
// spidx = e;
// }
// if ((spidx != -1))
// {
// //SizeF line1Size = g.MeasureString(EmotionLabels[epidx], font);
// //SizeF line2Size = g.MeasureString(SentimentLabels[spidx], font);
// //float width = Math.Max(line1Size.Width, line2Size.Width);
// //float offset = Math.Max(line2Size.Height, font.GetHeight());
// //float height = line1Size.Height + offset;
// //int x = data[0].rectangle.x + data[0].rectangle.w;
// //int y = data[0].rectangle.y > 0 ? data[0].rectangle.y : data[0].rectangle.h - (int)height;
// //if (x + Math.Max(line1Size.Width, line2Size.Width) > bitmap.Width)
// //{
// // x = data[0].rectangle.x - (int)width;
// //}
// //g.DrawString(EmotionLabels[epidx], font, brushTxt, x, y);
// //g.DrawString(SentimentLabels[spidx], font, brushTxt, x, data[0].rectangle.y > 0 ? y + offset : y - offset);
// }
// }
// }
// }
// // release objects
// pEmo.Dispose();
//}
manager.ReleaseFrame();
GC.Collect();
}
}
