/* Displaying current frames hand joints */
private void TrackJoints(PXCMHandData handOutput)
{
if (hasLaunched)
{
return;
}
//Get hand by time of appearence
if (handOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (handData.IsCalibrated() == false)
{
calibText.text = "Please wait Calibrating hand.";
return;
}
calibText.text = "Hand Calibrated!";
handData.QueryTrackedJoint(PXCMHandData.JointType.JOINT_THUMB_TIP, out ThumbJointData);
handData.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out IndexJointData);
/* get joint distance */
thumb_pos = new Vector3(ThumbJointData.positionWorld.x, ThumbJointData.positionWorld.y, ThumbJointData.positionWorld.z);
index_pos = new Vector3(IndexJointData.positionWorld.x, IndexJointData.positionWorld.y, IndexJointData.positionWorld.z);
pinch_dist = Vector3.Distance(index_pos, thumb_pos);
/* Update game based on pinch result*/
HandlePinchResult(pinch_dist);
}
}
private void RecogMove(PXCMHandData.IHand hand, PXCMPointF32 mcp)
{
int side = (int)hand.QueryBodySide() - 1;
if (side < 0)
{
return;
}
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out middleData[side]);
if (oldMiddleData[side] == null)
{
oldMiddleData[side] = middleData[side];
return;
}
//Console.WriteLine(middleData[side].speed.x);
var distance = Math.Pow(
Math.Pow(middleData[side].positionWorld.x - oldMiddleData[side].positionWorld.x, 2)
+ Math.Pow(middleData[side].positionWorld.y - oldMiddleData[side].positionWorld.y, 2)
+ Math.Pow((middleData[side].positionWorld.z - oldMiddleData[side].positionWorld.z) * 1000, 2),
0.5);
oldMiddleData[side] = middleData[side];
//Console.Write(middleData[side].positionImage.x + ":" + oldMiddleData[side].positionImage.x + ":");
//Console.WriteLine(middleData[side].positionImage.x - oldMiddleData[side].positionImage.x);
//oldMiddleData = middleData;
}
public void updateData(PXCMHandData.IHand handData)
{
this.handJoint = new PXCMHandData.JointData[22];
var jointTypes = Enum.GetValues(typeof(PXCMHandData.JointType)).Cast <PXCMHandData.JointType>();
int index = 0;
foreach (var jtmp in jointTypes)
{
PXCMHandData.JointData jdata = null;
handData.QueryTrackedJoint(jtmp, out jdata);
handJoint[index] = jdata;
index++;
}
index = 0;
this.handFinger = new PXCMHandData.FingerData[5];
var fingerTypes = Enum.GetValues(typeof(PXCMHandData.FingerType)).Cast <PXCMHandData.FingerType>();
foreach (var fmp in fingerTypes)
{
PXCMHandData.FingerData fdata = null;
handData.QueryFingerData(fmp, out fdata);
handFinger[index] = fdata;
index++;
}
this.handContourInner = new List <PXCMPointI32[]>();
this.handContourOuter = new List <PXCMPointI32[]>();
int contourNumber = handData.QueryNumberOfContours();
if (contourNumber > 0)
{
PXCMPointI32[] pointOuter;
PXCMPointI32[] pointInner;
for (int k = 0; k < contourNumber; ++k)
{
PXCMHandData.IContour contour;
pxcmStatus sts = handData.QueryContour(k, out contour);
if (sts == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (contour.IsOuter() == true)
{
contour.QueryPoints(out pointOuter);
handContourOuter.Add(pointOuter);
}
else
{
contour.QueryPoints(out pointInner);
handContourInner.Add(pointInner);
}
}
}
}
}
private void SetJointdata(PXCMHandData.IHand handInfo, PXCMHandData.JointType jointType, Item joint)
{
PXCMHandData.JointData jointData;
if (handInfo.QueryTrackedJoint(jointType, out jointData) != NoError)
{
joint.IsVisible = false;
return;
}
SetVisibleJointPosition(joint, jointData);
}
private Hand GetHandData(PXCMHandData.IHand currentHandData)
{
PXCMHandData.JointData jointData;
currentHandData.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out jointData);
var positionImage = jointData.positionWorld;
float z = positionImage.z;
var center = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_CENTER);
Hand hand = new Hand
{
Thumb = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_THUMB_BASE),
Center = center
},
Index = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_INDEX_BASE),
Center = center
},
Middle = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_MIDDLE_BASE),
Center = center
},
Ring = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_RING_BASE),
Center = center
},
Pinky = new Finger
{
Tip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_TIP),
BelowTip = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_JT2),
AboveBase = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_JT1),
Base = GetPointFromJoint(currentHandData, PXCMHandData.JointType.JOINT_PINKY_BASE),
Center = center
},
z = z
};
return(hand);
}
private static Point GetPointFromJoint(PXCMHandData.IHand currentHandData, PXCMHandData.JointType jointType)
{
PXCMHandData.JointData jointData;
currentHandData.QueryTrackedJoint(jointType, out jointData);
var positionImage = jointData.positionImage;
int x = (int)(WIDTH - positionImage.x);
int y = (int)positionImage.y;
return(new Point
{
X = x,
Y = y
});
}
/// <summary> 指のデータを取得する </summary>
private void GetFingerData(PXCMHandData.IHand hand, PXCMHandData.JointType jointType)
{
PXCMHandData.JointData jointData;
var sts = hand.QueryTrackedJoint(jointType, out jointData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
// Depth座標系をカラー座標系に変換する
var depthPoint = new PXCMPoint3DF32[1];
var colorPoint = new PXCMPointF32[1];
depthPoint[0].x = jointData.positionImage.x;
depthPoint[0].y = jointData.positionImage.y;
depthPoint[0].z = jointData.positionWorld.z * 1000;
projection.MapDepthToColor(depthPoint, colorPoint);
var masp = hand.QueryMassCenterImage();
var mdp = new PXCMPoint3DF32[1];
var mcp = new PXCMPointF32[1];
mdp[0].x = masp.x;
mdp[0].y = masp.y;
mdp[0].z = hand.QueryMassCenterWorld().z * 1000;
projection.MapDepthToColor(mdp, mcp);
//Console.WriteLine(mcp[0].x);
AddEllipse(new Point(mcp[0].x, mcp[0].y), 10, Brushes.Red, 1);
colorPoint = mcp;
//AddEllipse(new Point(colorPoint[0].x, colorPoint[0].y), 5, Brushes.White, 1);
if (ensembleTimer.IsEnabled)
{
DetectTap(hand, mcp[0]);
}
//if (ensembleTimer.IsEnabled) RecogMove(hand,mcp[0]);
//RecogMove(hand,mcp[0]);
}
pxcmStatus newHandFrame(PXCMHandModule hand)
{
if (hand != null)
{
PXCMHandData handData = hand.CreateOutput();
handData.Update();
PXCMHandData.IHand iHandDataLeft = null, iHandDataRight = null;
PXCMHandData.JointData jointData = null;
PXCMImage image = null;
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out iHandDataLeft);
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out iHandDataRight);
if (handForm != null && !handForm.IsDisposed)
{
this.handForm.HandCount = handData.QueryNumberOfHands();
if (iHandDataLeft != null)
{
iHandDataLeft.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.LeftHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
if (iHandDataRight != null)
{
iHandDataRight.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.RightHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
}
if (iHandDataLeft != null)
{
if (jointData == null)
{
iHandDataLeft.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (iHandDataRight != null)
{
if (jointData == null)
{
iHandDataRight.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (jointData != null && canTrack.Checked)
{
Cursor.Position = new System.Drawing.Point(
(int)((640.0f - jointData.positionImage.x) * Screen.PrimaryScreen.Bounds.Width / 640.0f),
(int)(jointData.positionImage.y * Screen.PrimaryScreen.Bounds.Height / 480.0f));
PXCMHandData.GestureData gestureData = null;
if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
Program.DoMouseClick();
}
Console.WriteLine("Z Position: " + jointData.positionWorld.z);
}
handData.Dispose();
}
return(pxcmStatus.PXCM_STATUS_NO_ERROR);
}
/* Displaying current frames hand joints */
private void TrackJoints(PXCMHandData handOutput)
{
//Get hand by time of appearence
if (handOutput.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMPoint4DF32 palmOrientation = handData.QueryPalmOrientation();
handData.QueryTrackedJoint(PXCMHandData.JointType.JOINT_THUMB_TIP, out ThumbJointData);
handData.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out IndexJointData);
handData.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out PalmCenterJointData);
/* get joint distance */
Vector3 thumb = new Vector3(ThumbJointData.positionWorld.x, ThumbJointData.positionWorld.y, ThumbJointData.positionWorld.z);
Vector3 index = new Vector3(IndexJointData.positionWorld.x, IndexJointData.positionWorld.y, IndexJointData.positionWorld.z);
Vector3 palm = new Vector3(PalmCenterJointData.positionWorld.x, PalmCenterJointData.positionWorld.y, PalmCenterJointData.positionWorld.z);
switch (rotationType)
{
case RotationType.FingerDelta:
//Rotate along y-axis
///Compare the difference between the z values and determine if is great than the threshold
float diff = CompareFloats(index.z, palm.z);
if (Mathf.Abs(diff) > 0.02f)
{
//begin to rotate using the difference thershold as a multiplier for rotation speed.
logo.transform.Rotate(Time.deltaTime * (diff * 500), 0, 0);
}
//rotate along x-axis
diff = CompareFloats(thumb.z, palm.z);
if (Mathf.Abs(diff) > 0.02f)
{
Debug.Log(diff.ToString());
//begin to rotate using the difference thershold as a multiplier for rotation speed.
logo.transform.Rotate(0, Time.deltaTime * (diff * 500), 0);
}
//rotate along x-axis
diff = CompareFloats(thumb.z, index.z);
if (Mathf.Abs(diff) > 0.02f)
{
Debug.Log(diff.ToString());
//begin to rotate using the difference thershold as a multiplier for rotation speed.
logo.transform.Rotate(0, 0, Time.deltaTime * (diff * 500));
}
break;
case RotationType.handleDelta:
//logo.transform.localRotation = new Quaternion(-palmOrientation.w, -palmOrientation.z, -palmOrientation.y, -palmOrientation.x);
logo.transform.Rotate(index.y * Time.deltaTime * 750, index.x * Time.deltaTime * 750, 0);
break;
case RotationType.Pinch:
pinch_dist = Vector3.Distance(index, thumb);
if (HandlePinchResult(pinch_dist))
{
//set the initial position
if (pinchPos == Vector3.zero)
{
pinchPos = index;
}
//get the vector between the initial position and the current index postion
Vector3 temp = pinchPos - index;
logo.transform.Rotate(-temp.y * Time.deltaTime * 750, -temp.x * Time.deltaTime * 750, -temp.z * Time.deltaTime * 750);
}
break;
default:
Debug.Log("something went wrong");
break;
}
}
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is TrackTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
// make sure we have valid values
if (RealWorldBoxDimensions.x <= 0)
{
RealWorldBoxDimensions.x = 1;
}
if (RealWorldBoxDimensions.y <= 0)
{
RealWorldBoxDimensions.y = 1;
}
if (RealWorldBoxDimensions.z <= 0)
{
RealWorldBoxDimensions.z = 1;
}
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
PXCMHandData.IHand data = null;
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0 &&
((ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
)
{
// Process Tracking
_uniqueID = data.QueryUniqueId();
PXCMHandData.JointData jointData;
data.QueryTrackedJoint(TrackedJoint, out jointData);
TrackTrigger specificTrigger = (TrackTrigger)trigger;
PXCMPoint3DF32 point = jointData.positionWorld;
Vector3 position = new Vector3(point.x, point.y, point.z);
position.x *= -100;
position.y *= 100;
position.z *= 100;
if (position.x + position.y + position.z == 0)
{
return(false);
}
TrackingUtilityClass.ClampToRealWorldInputBox(ref position, RealWorldBoxCenter, RealWorldBoxDimensions);
TrackingUtilityClass.Normalize(ref position, RealWorldBoxCenter, RealWorldBoxDimensions);
if (!float.IsNaN(position.x) && !float.IsNaN(position.y) && !float.IsNaN(position.z))
{
specificTrigger.Position = position;
}
else
{
return(false);
}
Quaternion q = new Quaternion(jointData.globalOrientation.x, jointData.globalOrientation.y,
jointData.globalOrientation.z, jointData.globalOrientation.w);
q = q * _zInvert * _yInvert;
specificTrigger.RotationQuaternion = q;
success = true;
}
}
else
{
return(false);
}
return(success);
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
Debug.LogError("Hand Analysis Module Not Set");
trigger.ErrorDetected = true;
return(false);
}
if (!(trigger is TranslationTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
// make sure we have valid values
if (RealWorldBoxDimensions.x <= 0)
{
RealWorldBoxDimensions.x = 1;
}
if (RealWorldBoxDimensions.y <= 0)
{
RealWorldBoxDimensions.y = 1;
}
if (RealWorldBoxDimensions.z <= 0)
{
RealWorldBoxDimensions.z = 1;
}
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
PXCMHandData.IHand data = null;
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0
&&
(
(ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
)
{
_uniqueID = data.QueryUniqueId();
// Process Translation
if (trigger is TranslationTrigger)
{
TranslationTrigger specificTrigger = (TranslationTrigger)trigger;
PXCMHandData.JointData jointData;
data.QueryTrackedJoint(TrackedJoint, out jointData);
Vector3 vecPos = new Vector3(-(jointData.positionWorld.x), (jointData.positionWorld.y), jointData.positionWorld.z);
vecPos.x *= 100;
vecPos.y *= 100;
vecPos.z *= 100;
TrackingUtilityClass.ClampToRealWorldInputBox(ref vecPos, RealWorldBoxCenter, RealWorldBoxDimensions);
TrackingUtilityClass.Normalize(ref vecPos, RealWorldBoxCenter, RealWorldBoxDimensions);
if (specificTrigger.Restart)
{
specificTrigger.Restart = false;
_referencePosition = vecPos;
return(false);
}
specificTrigger.Translation = vecPos - _referencePosition;
Vector3 passCriteria = new Vector3();
passCriteria.x = Mathf.Abs(vecPos.x) - Mathf.Abs(Thresholds.x);
passCriteria.y = Mathf.Abs(vecPos.y) - Mathf.Abs(Thresholds.y);
passCriteria.z = Mathf.Abs(vecPos.z) - Mathf.Abs(Thresholds.z);
if (passCriteria.x > 0 || passCriteria.y > 0 || passCriteria.z > 0)
{
_referencePosition = vecPos;
success = true;
}
}
}
}
else
{
return(false);
}
return(success);
}
public void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
hand = senseManager.QueryHand();
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
// Get number of tracked hands
nhands = handData.QueryNumberOfHands();
if (nhands > 0)
{
// Retrieve hand identifier
handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId);
// Retrieve hand data
handData.QueryHandDataById(handId, out ihand);
PXCMHandData.BodySideType bodySideType = ihand.QueryBodySide();
if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
leftHand = true;
}
else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
leftHand = false;
}
// Retrieve all hand joint data
for (int i = 0; i < nhands; i++)
{
for (int j = 0; j < 0x20; j++)
{
PXCMHandData.JointData jointData;
ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
nodes[i][j] = jointData;
}
}
// Get world coordinates for tip of middle finger on the first hand in camera range
handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x;
handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y;
handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z;
swipehandTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.x;
swipehandTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.y;
swipehandTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.z;
//Console.Out.WriteLine("Before x={0}", swipehandTipX);
//Console.Out.WriteLine("Before speed={0}", nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].speed.x);
// Retrieve gesture data
if (handData.IsGestureFired("spreadfingers", out gestureData))
{
gesture = Gesture.FingerSpread;
}
else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
gesture = Gesture.Pinch;
}
else if (handData.IsGestureFired("wave", out gestureData))
{
gesture = Gesture.Wave;
}
else if (handData.IsGestureFired("swipe_left", out gestureData))
{
gesture = Gesture.SwipeLeft;
}
else if (handData.IsGestureFired("swipe_right", out gestureData))
{
gesture = Gesture.SwipeRight;
}
else if (handData.IsGestureFired("fist", out gestureData))
{
gesture = Gesture.Fist;
}
else if (handData.IsGestureFired("thumb_up", out gestureData))
{
gesture = Gesture.Thumb;
}
}
else
{
gesture = Gesture.Undefined;
}
UpdateUI();
if (handData != null)
{
handData.Dispose();
}
}
senseManager.ReleaseFrame();
}
}
private void DetectTap(PXCMHandData.IHand hand, PXCMPointF32 mcp)
{
PXCMHandData.JointData midData;
PXCMHandData.JointData cntData;
//指のデータをとってくる(depth)
//ユーザの右手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out cntData);
RightCenter = cntData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out midData);
RightMiddle = midData.positionWorld;
}
//ユーザの左手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out cntData);
LeftCenter = cntData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out midData);
LeftMiddle = midData.positionWorld;
}
if (mcp.y < 0)
{
mcp.y = 0;
}
if (mcp.y > ColorImage.Height)
{
mcp.y = (float)ColorImage.Height;
}
//Console.WriteLine(rsw.ElapsedMilliseconds);
//if文の条件を記述(前の指のデータと比較)
// ユーザの右手でタップ
if (-RightMiddle.z + preRightMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
Math.Pow(Math.Pow(RightMiddle.x - preRightMiddle.x, 2) // 指先の速度が1.8m/s以上
+ Math.Pow(RightMiddle.y - preRightMiddle.y, 2)
+ Math.Pow(RightMiddle.z * 1000 - preRightMiddle.z * 1000, 2), 0.5) > 0.03 &&
Math.Pow(Math.Pow(RightCenter.x - preRightCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ Math.Pow(RightCenter.y - preRightCenter.y, 2)
+ Math.Pow(RightCenter.z * 1000 - preRightCenter.z * 1000, 2), 0.5) > 0.01 &&
rsw.ElapsedMilliseconds > 300
)
{
//tap音を出力
midi.OnNote(currentFreqs[(int)(((ColorImage.Height - mcp.y) / (ColorImage.Height + 0.01)) * currentFreqs.Length)]);
rsw.Restart();
}
//var c = ColorImage.Height - y;
//var e = c / ColorImage.Height;
//var d = e * currentFreqs.Length;
//Index.Text = y + "\n" + d + "\n" + (int)d;
//Index.Text = mcp.y.ToString();
//Chord.Text = e.ToString();
// ユーザの左手でタップ
if (-LeftMiddle.z + preLeftMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
Math.Pow(Math.Pow(LeftMiddle.x - preLeftMiddle.x, 2) // 指先の速度が1.8m/s以上
+ Math.Pow(LeftMiddle.y - preLeftMiddle.y, 2)
+ Math.Pow(LeftMiddle.z * 1000 - preLeftMiddle.z * 1000, 2), 0.5) > 0.03 &&
Math.Pow(Math.Pow(LeftCenter.x - preLeftCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ Math.Pow(LeftCenter.y - preLeftCenter.y, 2)
+ Math.Pow(LeftCenter.z * 1000 - preLeftCenter.z * 1000, 2), 0.5) > 0.01 &&
lsw.ElapsedMilliseconds > 300
)
{
//tap音を出力
//midi.OnNote(currentFreqs[(int)(((ColorImage.Height - mcp.y) / (ColorImage.Height + 0.01)) * currentFreqs.Length)]);
lsw.Restart();
}
//Console.WriteLine("RightCenter.x:" + RightCenter.x);
//Console.WriteLine("preRightCenter.x:" + preRightCenter.x);
//Console.WriteLine();
//Console.WriteLine("RightMiddle.x:" + RightMiddle.x);
//Console.WriteLine("preRightMiddle.x:" + preRightMiddle.x);
//前の指のデータに今の指のデータを上書き
//plc,preLeftMiddle,preRightCenter,preRightMiddle
// 上手くいかなければディープコピーする．
preRightCenter.x = RightCenter.x;
preRightCenter.y = RightCenter.y;
preRightCenter.z = RightCenter.z;
preRightMiddle.x = RightMiddle.x;
preRightMiddle.y = RightMiddle.y;
preRightMiddle.z = RightMiddle.z;
preLeftCenter.x = LeftCenter.x;
preLeftCenter.y = LeftCenter.y;
preLeftCenter.z = LeftCenter.z;
preLeftMiddle.x = LeftMiddle.x;
preLeftMiddle.y = LeftMiddle.y;
preLeftMiddle.z = LeftMiddle.z;
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Hand))
{
trigger.ErrorDetected = true;
Debug.LogError("Hand Analysis Module Not Set");
return(false);
}
if (!(trigger is EventTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
//AcquireFrame
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.HandDataOutput != null)
{
PXCMHandData.IHand data = null;
if (SenseToolkitManager.Instance.HandDataOutput.QueryNumberOfHands() > 0
&&
(
(ContinuousTracking && SenseToolkitManager.Instance.HandDataOutput.QueryHandDataById(_uniqueID, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
||
SenseToolkitManager.Instance.HandDataOutput.QueryHandData(WhichHand, HandIndex, out data) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
)
{
_uniqueID = data.QueryUniqueId();
PXCMHandData.JointData jointD;
data.QueryTrackedJoint(PXCMHandData.JointType.JOINT_PINKY_JT2, out jointD);
if (jointD.confidence < 100)
{
return(false);
}
if (data.QueryOpenness() <= OpennessFactor)
{
_lastFrameDetected = true;
}
else
{
if (_lastFrameDetected)
{
success = true;
}
_lastFrameDetected = false;
}
}
else
{
if (_lastFrameDetected)
{
success = true;
}
_lastFrameDetected = false;
}
}
return(success);
}
private void DetectTap(PXCMHandData.IHand hand)
{
PXCMHandData.JointData MiddleData;
PXCMHandData.JointData CenterData;
//指のデータをとってくる(depth)
//ユーザの右手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out CenterData);
RightCenter = CenterData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out MiddleData);
RightMiddle = MiddleData.positionWorld;
//RightCenter = hand.QueryMassCenterWorld();
}
//ユーザの左手のデータ
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_CENTER, out CenterData);
LeftCenter = CenterData.positionWorld;
hand.QueryTrackedJoint(PXCMHandData.JointType.JOINT_MIDDLE_TIP, out MiddleData);
LeftMiddle = MiddleData.positionWorld;
}
//if文の条件を記述(前の指のデータと比較)
// ユーザの右手でタップ
if (-RightMiddle.z + preRightMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
System.Math.Pow(System.Math.Pow(RightMiddle.x - preRightMiddle.x, 2) // 指先の速度が1.8m/s以上
+ System.Math.Pow(RightMiddle.y - preRightMiddle.y, 2)
+ System.Math.Pow(RightMiddle.z * 1000 - preRightMiddle.z * 1000, 2), 0.5) > 0.03 &&
System.Math.Pow(System.Math.Pow(RightCenter.x - preRightCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ System.Math.Pow(RightCenter.y - preRightCenter.y, 2)
+ System.Math.Pow(RightCenter.z * 1000 - preRightCenter.z * 1000, 2), 0.5) > 0.01 &&
rsw.ElapsedMilliseconds > 250
)
{
//tap音を出力
midiManager.SetOnNote(player.MusicTime);
rsw.Restart();
}
// ユーザの左手でタップ
if (-LeftMiddle.z + preLeftMiddle.z > 0.02 && // 1F(約1/60秒)あたりの深度の変化が0.02m以上
System.Math.Pow(System.Math.Pow(LeftMiddle.x - preLeftMiddle.x, 2) // 指先の速度が1.8m/s以上
+ System.Math.Pow(RightMiddle.y - preLeftMiddle.y, 2)
+ System.Math.Pow(RightMiddle.z - preLeftMiddle.z, 2), 0.5) > 0.03 &&
System.Math.Pow(System.Math.Pow(RightCenter.x - preLeftCenter.x, 2) // 手のひらの速度が0.6m/s以上
+ System.Math.Pow(RightCenter.y - preLeftCenter.y, 2)
+ System.Math.Pow(RightCenter.z - preLeftCenter.z, 2), 0.5) > 0.01 &&
System.Math.Pow(System.Math.Pow(RightCenter.x - preLeftCenter.x, 2) // 手のひらの速度が1.5m/s以下
+ System.Math.Pow(RightCenter.y - preLeftCenter.y, 2)
+ System.Math.Pow(RightCenter.z - preLeftCenter.z, 2), 0.5) < 0.025
)
{
//tap音を出力
midiManager.SetOnNote(player.MusicTime);
}
//Console.WriteLine("RightCenter.x:" + RightCenter.x);
//Console.WriteLine("preRightCenter.x:" + preRightCenter.x);
//Console.WriteLine();
//Console.WriteLine("RightMiddle.x:" + RightMiddle.x);
//Console.WriteLine("preRightMiddle.x:" + preRightMiddle.x);
//前の指のデータに今の指のデータを上書き
//plc,preLeftMiddle,preRightCenter,preRightMiddle
// 上手くいかなければディープコピーする．
preRightCenter.x = RightCenter.x;
preRightCenter.y = RightCenter.y;
preRightCenter.z = RightCenter.z;
preRightMiddle.x = RightMiddle.x;
preRightMiddle.y = RightMiddle.y;
preRightMiddle.z = RightMiddle.z;
preLeftCenter.x = LeftCenter.x;
preLeftCenter.y = LeftCenter.y;
preLeftCenter.z = LeftCenter.z;
preLeftMiddle.x = LeftMiddle.x;
preLeftMiddle.y = LeftMiddle.y;
preLeftMiddle.z = LeftMiddle.z;
}
/// <summary> 指のデータを取得する </summary>
private bool GetFingerData(PXCMHandData.IHand hand, PXCMHandData.JointType jointType)
{
PXCMHandData.JointData jointData;
var sts = hand.QueryTrackedJoint(jointType, out jointData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return(false);
}
// Depth座標系をカラー座標系に変換する
var depthPoint = new PXCMPoint3DF32[1];
var colorPoint = new PXCMPointF32[1];
depthPoint[0].x = jointData.positionImage.x;
depthPoint[0].y = jointData.positionImage.y;
depthPoint[0].z = jointData.positionWorld.z * 1000;
projection.MapDepthToColor(depthPoint, colorPoint);
var masp = hand.QueryMassCenterImage();
var mdp = new PXCMPoint3DF32[1];
var mcp = new PXCMPointF32[1];
mdp[0].x = masp.x;
mdp[0].y = masp.y;
mdp[0].z = hand.QueryMassCenterWorld().z * 1000;
projection.MapDepthToColor(mdp, mcp);
//Console.WriteLine(mcp[0].x);
AddEllipse(new Point(mcp[0].x, mcp[0].y), 10, Brushes.Red, 1);
colorPoint = mcp;
//ユーザの右手に対して演奏領域の当たり判定確認
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
for (int i = 0; i < 5; i++)
{
if ((imageColor.Height / 5) * i <= colorPoint[0].y && colorPoint[0].y < (imageColor.Height / 5) * (i + 1))
{
if (16 - i != NowRange)
{
NowRange = 16 - i;
PivotList.Dispatcher.BeginInvoke(
new Action(() =>
{
PivotList.SelectedItem = NowRange;
}
));
}
}
}
}
//ユーザの左手に対してアイコンの当たり判定の確認
if (hand.QueryBodySide() == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
IconHitCheck(colorPoint[0]);
}
AddEllipse(new Point(colorPoint[0].x, colorPoint[0].y), 5, Brushes.White, 1);
return(true);
}
