/// <summary> 手のデータを更新する </summary>
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// データを初期化する
CanvasFaceParts.Children.Clear();
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
GetFingerData(hand, PXCMHandData.JointType.JOINT_MIDDLE_TIP);
DetectTap(hand);
}
}
/// <summary> 手のデータを更新する </summary>
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// データを初期化する
PartsCanvas.Children.Clear();
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
//Console.WriteLine(hand.QueryUniqueId());
int side = (int)hand.QueryBodySide();
if (side > 0)
{
side2id[side - 1] = hand.QueryUniqueId();
}
GetFingerData(hand, PXCMHandData.JointType.JOINT_MIDDLE_TIP);
}
}
// Update is called once per frame
void Update()
{
if (sm != null)
{
/* Wait until any frame data is available */
if (sm.AcquireFrame(false, 0) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
/* Retrieve latest extremity data */
if (hand_data.Update() == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
;
}
{
TrackExtremity(hand_data);
if (hand_data.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (handData.QueryExtremityPoint(PXCMHandData.ExtremityType.EXTREMITY_LEFTMOST, out extremityPoint) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.Log("LeftMost Extremity Position = " + extremityPoint.pointWorld);
}
}
}
/* Now, release the current frame so we can process the next frame */
sm.ReleaseFrame();
}
}
}
// Update is called once per frame
void Update()
{
if (sm != null)
{
/* Wait until any frame data is available */
if (sm.AcquireFrame(false, 0) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (hand_data.QueryNumberOfHands() == 0)
{
calibText.text = "";
}
/* Retrieve latest hand data, only update slingshot if ready */
if (hand_data.Update() == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
;
}
{
TrackJoints(hand_data);
pDrag.UpdateDrag(isPinching);
}
/* Now, release the current frame so we can process the next frame */
sm.ReleaseFrame();
}
}
}
void Update()
{
if (mRS.AcquireFrame(false) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
mHand.Update();
getCursorPositions();
mRS.ReleaseFrame();
}
}
// 手のデータを更新する
private void UpdateHandFrame()
{
handData.Update();
// 画像の初期化
ImageHand1.Source = ImageHand2.Source = null;
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 手の画像を取得する
PXCMImage image;
sts = hand.QuerySegmentationImage(out image);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// マスク画像を取得する
PXCMImage.ImageData data;
sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ,
PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out data);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// マスク画像のサイズはDepthに依存
// 手は2つまで
var info = image.QueryInfo();
var bitmap = BitmapSource.Create(info.width, info.height, 96, 96,
PixelFormats.Gray8, null, data.planes[0],
data.pitches[0] * info.height, data.pitches[0]);
if (i == 0)
{
ImageHand1.Source = bitmap;
}
else
{
ImageHand2.Source = bitmap;
}
image.ReleaseAccess(data);
}
}
// Update is called once per frame
void Update()
{
/* Make sure PXCMSenseManager Instance is Initialized */
if (SenseToolkitManager.Instance.SenseManager == null)
{
return;
}
/* Wait until any frame data is available true(aligned) false(unaligned) */
if (SenseToolkitManager.Instance.SenseManager.AcquireFrame(false, 0) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve am instance of hand tracking Module */
handAnalyzer = SenseToolkitManager.Instance.SenseManager.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve an instance of hand tracking Data */
PXCMHandData _outputData = handAnalyzer.CreateOutput();
if (_outputData != null)
{
_outputData.Update();
//Retrieve joint data, gesture recognition data and alert notification data
//refer to next section
//AcquireFrame
/* Retrieve Gesture Data */
PXCMHandData.GestureData _gestureData;
for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++)
{
if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
//Display the gestures: explained in rendering the frame section
if ((_gestureData.name == "spreadfingers") || (_gestureData.name == "tap") ||
(_gestureData.name == "swipe"))
{
CursorController.isHandClicked = false;
}
else if ((_gestureData.name == "fist") || (_gestureData.name == "full_pinch") ||
(_gestureData.name == "thumb_down") || (_gestureData.name == "thumb_up") ||
(_gestureData.name == "two_fingers_pinch_open") || (_gestureData.name == "v_sign"))
{
CursorController.isHandClicked = true;
}
else if (_gestureData.name == "wave")
{
transform.position = new Vector3(0, 0, 0);
}
}
}
}
}
/* Realease the frame to process the next frame */
SenseToolkitManager.Instance.SenseManager.ReleaseFrame();
}
// 手のデータを更新する
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// データを初期化する
CanvasFaceParts.Children.Clear();
// 点に色を付ける
Brush[] colors = new Brush[] {
Brushes.Blue,
Brushes.Green,
Brushes.Red,
Brushes.Yellow,
Brushes.Purple,
Brushes.LightBlue,
};
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 指の関節を列挙する
for (int j = 0; j < PXCMHandData.NUMBER_OF_EXTREMITIES; j++)
{
PXCMHandData.ExtremityData extremityData;
sts = hand.QueryExtremityPoint((PXCMHandData.ExtremityType)j,
out extremityData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
AddEllipse(CanvasFaceParts,
new Point(extremityData.pointImage.x, extremityData.pointImage.y),
10, colors[j], -1);
}
}
}
public override void Work(Graphics g)
{
data = module.CreateOutput();
data.Update();
PXCMHandData.IHand hands = null;
data.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, 0, out hands);
if (hands == null)
{
return;
}
PXCMRectI32 rect = hands.QueryBoundingBoxImage();
Rectangle rectangle = new Rectangle(rect.x, rect.y, rect.w, rect.h); // Convert to Rectangle
g.DrawRectangle(pen, rectangle); // Draw
}
public void Loop(LoopObjects loopObjects)
{
_loopObjects = loopObjects;
var handModule = _camera.Manager.QueryHand();
if (handModule == null)
{
return;
}
_handData = handModule.CreateOutput();
_handData.Update();
TrackHand(_camera.LeftHand, PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS);
TrackHand(_camera.RightHand, PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS);
}
// 手のデータを更新する
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// データを初期化する
CanvasFaceParts.Children.Clear();
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 指の関節を列挙する
for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++)
{
// 指のデータを取得する
PXCMHandData.JointData jointData;
sts = hand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// Depth座標系をカラー座標系に変換する
var depthPoint = new PXCMPoint3DF32[1];
var colorPoint = new PXCMPointF32[1];
depthPoint[0].x = jointData.positionImage.x;
depthPoint[0].y = jointData.positionImage.y;
depthPoint[0].z = jointData.positionWorld.z * 1000;
projection.MapDepthToColor(depthPoint, colorPoint);
AddEllipse(CanvasFaceParts,
new Point(colorPoint[0].x, colorPoint[0].y),
5, Brushes.Green);
}
}
}
public override void Work(Graphics g)
{
// Retrieve gesture data
hand = senseManager.QueryHand();
face = senseManager.QueryFace(); //
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
handWaving = handData.IsGestureFired("wave", out gestureData);
}
if (face != null)
{
faceData = face.CreateOutput();
faceData.Update();
//surching faces
Int32 nfaces = faceData.QueryNumberOfDetectedFaces();
for (Int32 i = 0; i < nfaces; i++)
{
// Retrieve the data instance
PXCMFaceData.Face faceI = faceData.QueryFaceByIndex(i);
PXCMFaceData.ExpressionsData edata = faceI.QueryExpressions();
if (edata != null)
{
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out score);
if (score.intensity >= 25)
{
smiling = true;
}
else
{
smiling = false;
}
Console.WriteLine(i + ": " + score.intensity);
}
}
faceData.Dispose();
}
}
// 手のデータを更新する
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// データを初期化する
CanvasFaceParts.Children.Clear();
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 指の関節を列挙する
for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++)
{
PXCMHandData.JointData jointData;
sts = hand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
AddEllipse(CanvasFaceParts,
new Point(jointData.positionImage.x, jointData.positionImage.y),
5, Brushes.Green);
}
}
}
void Update()
{
if (gestureCooldown > 0)
{
gestureCooldown -= Time.deltaTime;
}
if (psm == null)
{
return;
}
if (psm.AcquireFrame(true) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
drawCameraVision();
handData.Update();
if (currentEffect == 2)
{
PXCMHandData.JointData[,] nodes = getHandInfo();
followJoints(nodes);
}
else if (currentEffect == 3)
{
// point cloud
createDepthMesh();
}
moveHands();
handleGestures();
// Release the frame
psm.ReleaseFrame();
}
private void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
// Get color image data
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Retrieve gesture data
hand = senseManager.QueryHand();
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
handWaving = handData.IsGestureFired("v_sign", out gestureData);
}
// Update the user interface
UpdateUI(colorBitmap);
// Release the frame
if (handData != null)
{
handData.Dispose();
}
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
senseManager.ReleaseFrame();
}
}
// Update is called once per frame
void Update()
{
handData = SenseToolkitManager.Instance.HandDataOutput;
if (handData != null)
{
handData.Update();
for (int i = 0; i < handData.QueryFiredGesturesNumber(); i++)
{
if (handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
DisplayGestures(gestureData);
}
}
PXCMHandData.AlertData alertData;
for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++)
{
if (handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
ProcessAlerts(alertData);
}
}
}
}
// 手のデータを更新する
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// ピクセルデータを初期化する
Array.Clear(imageBuffer, 0, imageBuffer.Length);
CanvasFaceParts.Children.Clear();
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 手の画像を取得する
PXCMImage image;
sts = hand.QuerySegmentationImage(out image);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// マスク画像を取得する
PXCMImage.ImageData data;
sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ,
PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out data);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 手の左右を取得する
var side = (int)hand.QueryBodySide();
// 手の開閉度(0-100)を取得する
var openness = hand.QueryOpenness();
// マスク画像のサイズはDepthに依存
// 手は2つまで
var info = image.QueryInfo();
// マスク画像をバイト列に変換する
var buffer = data.ToByteArray(0, data.pitches[0] * info.height);
for (int j = 0; j < info.height * info.width; ++j)
{
if (buffer[j] != 0)
{
var index = j * BYTE_PER_PIXEL;
// 手の左右および手の開閉度で色合いを決める
// 左手=1：0-127の範囲
// 右手=2：0-254の範囲
var value = (byte)((side * 127) * (openness / 100.0f));
imageBuffer[index + 0] = value;
imageBuffer[index + 1] = value;
imageBuffer[index + 2] = value;
}
}
image.ReleaseAccess(data);
// 手の重心を表示する
var center = hand.QueryMassCenterImage();
var ellipse = new Ellipse()
{
Width = 5,
Height = 5,
Fill = Brushes.Blue,
};
Canvas.SetLeft(ellipse, center.x);
Canvas.SetTop(ellipse, center.y);
CanvasFaceParts.Children.Add(ellipse);
// 手の範囲を表示する
var boundingbox = hand.QueryBoundingBoxImage();
var rectangle = new Rectangle()
{
Width = boundingbox.w,
Height = boundingbox.h,
Stroke = Brushes.Red,
StrokeThickness = 2,
};
Canvas.SetLeft(rectangle, boundingbox.x);
Canvas.SetTop(rectangle, boundingbox.y);
CanvasFaceParts.Children.Add(rectangle);
}
// ピクセルデータを更新する
imageBitmap.WritePixels(imageRect, imageBuffer,
DEPTH_WIDTH * BYTE_PER_PIXEL, 0);
}
public static void start(IoT_RealSense_Surfing frm)
{
keepLooping = true;
iot_form = frm;
InitializeMqqtClient();
PXCMSession session = PXCMSession.CreateInstance();
// Querying the SDK version
Console.WriteLine(frm.comboTarget.SelectedText);
if (session != null)
{
// Optional steps to send feedback to Intel Corporation to understand how often each SDK sample is used.
PXCMMetadata md = session.QueryInstance <PXCMMetadata>();
if (md != null)
{
string sample_name = "Emotion Viewer CS";
md.AttachBuffer(1297303632, System.Text.Encoding.Unicode.GetBytes(sample_name));
}
//Application.Run(new MainForm(session));
//session.Dispose();
}
//PXCMSession.ImplVersion version = session.QueryVersion();
//Console.WriteLine("RealSense SDK Version {0}.{1}", version.major, version.minor);
session.CreateImpl <PXCMRotation>(out rotationHelper);
// Creating the SenseManager
PXCMSenseManager senseManager = session.CreateSenseManager();
if (senseManager == null)
{
Console.WriteLine("Failed to create the SenseManager object.");
return;
}
// Enabling Emotion Module
pxcmStatus enablingModuleStatus = senseManager.EnableEmotion();
if (enablingModuleStatus != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("Failed to enable the Emotion Module");
return;
}
// Getting the instance of the Emotion Module
PXCMEmotion emotionModule = senseManager.QueryEmotion();
if (emotionModule == null)
{
Console.WriteLine("Failed to query the emotion module");
return;
}
// Initializing the camera
if (senseManager.Init() < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("Failed to initialize the SenseManager");
return;
}
// Enabling the Hand module
pxcmStatus enablingModuleStatus1 = senseManager.EnableHand("Hand Module");
if (enablingModuleStatus1 != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("Failed to enable the Hand Module");
return;
}
// Getting the instance of the Hand Module
PXCMHandModule handModule = senseManager.QueryHand();
if (handModule == null)
{
Console.WriteLine("Failed to get the HandModule object.");
return;
}
// Creating an active configuration
PXCMHandConfiguration handConfiguration = handModule.CreateActiveConfiguration();
if (handConfiguration == null)
{
Console.WriteLine("Failed to create the HandConfiguration object.");
return;
}
// Listing the available gestures
int supportedGesturesCount = handConfiguration.QueryGesturesTotalNumber();
if (supportedGesturesCount > 0)
{
Console.WriteLine("Supported gestures:");
for (int i = 0; i < supportedGesturesCount; i++)
{
string gestureName = string.Empty;
if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("\t" + gestureName);
}
}
}
// Enabling some gestures
String[] enabledGestures = { GESTURE_CLICK, GESTURE_VSIGN, GESTURE_FIST, GESTURE_SPREADFINGERS };
foreach (String gesture in enabledGestures)
{
if (!handConfiguration.IsGestureEnabled(gesture))
{
handConfiguration.EnableGesture(gesture);
}
}
handConfiguration.ApplyChanges();
// Creating a data output object
PXCMHandData handData = handModule.CreateOutput();
if (handData == null)
{
Console.WriteLine("Failed to create the HandData object.");
return;
}
// Initializing the SenseManager
if (senseManager.Init() != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine(senseManager.Init());
return;
}
// Looping to query the hands information
while (keepLooping)
{
// Acquiring a frame
if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
break;
}
// Updating the hand data
if (handData != null)
{
handData.Update();
}
//ProcessHands(handData);
ProcessGestures(handData);
ProcessEmotions(emotionModule);
// Releasing the acquired frame
senseManager.ReleaseFrame();
/* using another frame to process different stuff? may be...
* // Acquiring a frame
* if (senseManager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
* {
* break;
* }
*
* // Processing Emotions
* ProcessEmotions(emotionModule);
*
* // Releasing the acquired frame
* senseManager.ReleaseFrame();*/
}
// Releasing resources
if (handData != null)
{
handData.Dispose();
}
if (handConfiguration != null)
{
handConfiguration.Dispose();
}
rotationHelper.Dispose();
senseManager.Close();
senseManager.Dispose();
session.Dispose();
client.Disconnect();
}
void Update()
{
//Dynamically Pause/Enable Modules
int numberOfEnabledModules = 0;
foreach (var option in _senseOptions)
{
if (option.RefCounter == 0 && option.Enabled)
{
if (option.ModuleCUID > 0)
{
SenseManager.PauseModule(option.ModuleCUID, true);
}
option.Enabled = false;
}
else if (option.RefCounter > 0 && !option.Enabled)
{
if (!option.Initialized)
{
OnDisable();
OnEnable();
Start();
}
if (option.ModuleCUID > 0)
{
SenseManager.PauseModule(option.ModuleCUID, false);
}
option.Enabled = true;
}
if (option.Enabled)
{
numberOfEnabledModules++;
}
}
//Update Speech commands if changed
if (_speechCommandsChanged)
{
UpdateSpeechCommands();
SpeechManager.Reset();
}
// Every frame update all the data
if (Initialized && numberOfEnabledModules > 0)
{
_sts = SenseManager.AcquireFrame(true, 0);
if (_sts == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled)
{
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.color != null)
{
ImageRgbOutput = _captureSample.color;
ImageRgbOutput.AddRef();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled ||
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled)
{
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.depth != null)
{
ImageDepthOutput = _captureSample.depth;
ImageDepthOutput.AddRef();
/* GZ
* if (!_isInitBlob)
* {
* PXCMImage.ImageInfo info = ImageDepthOutput.QueryInfo();
* BlobExtractor.Init(info);
* BlobExtractor.SetMaxBlobs(MaxBlobsToDetect);
* _isInitBlob = true;
* }
*/
if (PointCloud == null)
{
PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled)
{
if (PointCloud == null)
{
PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
_sts = Projection.QueryVertices(ImageDepthOutput, PointCloud);
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.UVMap).Enabled)
{
if (UvMap == null)
{
UvMap = new PXCMPointF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
Projection.QueryUVMap(ImageDepthOutput, UvMap);
}
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled)
{
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.ir != null)
{
ImageIROutput = _captureSample.ir;
ImageIROutput.AddRef();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled)
{
if (Image3DSegmentationOutput != null)
{
Image3DSegmentationOutput.Dispose();
}
PXCM3DSeg seg = SenseManager.Query3DSeg();
if (seg != null)
{
Image3DSegmentationOutput = seg.AcquireSegmentedImage();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled)
{
FaceModuleOutput.Update();
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled)
{
HandDataOutput.Update();
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled)
{
BlobDataOutput.Update();
}
_captureSample = null;
SenseManager.ReleaseFrame();
}
//Speech
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled)
{
SpeechManager.QueryRecognizedCommands(out SpeechOutput);
}
}
}
// Update is called once per frame
void Update()
{
/* Make sure SenseManager Instance is valid */
if (sm == null)
{
return;
}
/* Wait until any frame data is available */
if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve hand tracking Module Instance */
handAnalyzer = sm.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput();
if (_handData != null)
{
_handData.Update();
/* Retrieve Gesture Data to manipulate GUIText */
PXCMHandData.GestureData gestureData;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++)
{
if (_handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
DisplayGestures(gestureData);
}
}
/* Retrieve Alert Data to manipulate GUIText */
PXCMHandData.AlertData alertData;
for (int i = 0; i < _handData.QueryFiredAlertsNumber(); i++)
{
if (_handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
ProcessAlerts(alertData);
}
}
/* Retrieve all joint Data */
for (int i = 0; i < _handData.QueryNumberOfHands(); i++)
{
PXCMHandData.IHand _iHand;
if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
for (int j = 0; j < MaxJoints; j++)
{
if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i][j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
jointData[i][j] = null;
}
}
if (!handList.ContainsKey(_iHand.QueryUniqueId()))
{
handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide());
}
}
}
/* Smoothen and Display the Data - Joints and Bones*/
DisplayJoints();
}
handAnalyzer.Dispose();
}
sm.ReleaseFrame();
RotateCam();
}
// Update is called once per frame
void Update()
{
if (session == null)
{
return;
}
// For accessing hand data
handAnalyzer = session.QueryHand();
faceAnalyzer = session.QueryFace();
if (handAnalyzer != null)
{
PXCMHandData handData = handAnalyzer.CreateOutput();
if (handData != null)
{
handData.Update();
PXCMHandData.IHand IHAND; // Ihand instance for accessing future data
// Int32 IhandData; // for QueryOpenness Value
// PXCMPoint3DF32 location; // Stores hand tracking position
//Fills IHAND with information to later be grabbed and used for tracking + openness
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, 0, out IHAND);
// If there is data in Ihand
if (IHAND != null)
{
// Debug.DrawLine(transform.position, hit.point, Color.red);
// Inits hand tracking from the center of the hand.
// location = IHAND.QueryMassCenterWorld();
// if (mCurrentDart != null)
// {
// Vector3 locationUnity = new Vector3(location.x, location.y, location.z);
// mCurrentDart.transform.localPosition = locationUnity * RSScale;
// }
}
}
handAnalyzer.Dispose();
session.ReleaseFrame();
}
if (faceAnalyzer != null)
{
PXCMFaceData facedata = faceAnalyzer.CreateOutput();
if (facedata != null)
{
Int32 nfaces = facedata.QueryNumberOfDetectedFaces();
for (Int32 i = 0; i < nfaces; i++)
{
// Retrieve the face landmark data instance
PXCMFaceData.Face face = facedata.QueryFaceByIndex(i);
PXCMFaceData.PoseData pdata = face.QueryPose();
// retrieve the pose information
PXCMFaceData.PoseEulerAngles angles;
pdata.QueryPoseAngles(out angles);
Debug.Log("Eular Angles yaw : " + angles.yaw);
Debug.Log("Eular Angles pitch: " + angles.pitch);
Debug.Log("Eular Angles Roll: " + angles.roll);
angles.pitch = gameObject.transform.rotation.z;
angles.yaw = gameObject.transform.rotation.y;
}
// device is a PXCMCapture.Device instance
}
}
}
// 手のデータを更新する
private void UpdateHandFrame()
{
// 手のデータを更新する
handData.Update();
// ピクセルデータを初期化する
Array.Clear(imageBuffer, 0, imageBuffer.Length);
// 検出した手の数を取得する
var numOfHands = handData.QueryNumberOfHands();
for (int i = 0; i < numOfHands; i++)
{
// 手を取得する
PXCMHandData.IHand hand;
var sts = handData.QueryHandData(
PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_ID, i, out hand);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// 手の画像を取得する
PXCMImage image;
sts = hand.QuerySegmentationImage(out image);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// マスク画像を取得する
PXCMImage.ImageData data;
sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ,
PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out data);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// マスク画像のサイズはDepthに依存
// 手は2つまで
var info = image.QueryInfo();
// マスク画像をバイト列に変換する
var buffer = data.ToByteArray(0, data.pitches[0] * info.height);
for (int j = 0; j < info.height * info.width; ++j)
{
if (buffer[j] != 0)
{
var index = j * BYTE_PER_PIXEL;
// 手のインデックスで色を決める
// ID=0：127
// ID=1：254
var value = (byte)((i + 1) * 127);
imageBuffer[index + 0] = value;
imageBuffer[index + 1] = value;
imageBuffer[index + 2] = value;
}
}
image.ReleaseAccess(data);
}
// ピクセルデータを更新する
imageBitmap.WritePixels(imageRect, imageBuffer,
DEPTH_WIDTH * BYTE_PER_PIXEL, 0);
}
public void HandPipeLine()
{
PXCMSenseManager pp = m_form.Session.CreateSenseManager();
if (pp == null)
{
throw new Exception("PXCMSenseManager null");
}
pp.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 360);
//手 初始化
PXCMHandModule handAnalysis;
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler();
handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
PXCMHandConfiguration handConfiguration = null;
PXCMHandData handData = null;
pxcmStatus status = pp.EnableHand();
handAnalysis = pp.QueryHand();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null)
{
Console.WriteLine("hand module load failed");
return;
}
handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
Console.WriteLine("Failed Create Configuration");
return;
}
handData = handAnalysis.CreateOutput();
if (handData == null)
{
Console.WriteLine("Failed Create Output");
return;
}
if (pp.Init(handler) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine("init failed");
return;
}
if (handConfiguration != null)
{
PXCMHandData.TrackingModeType trackingMode = PXCMHandData.TrackingModeType.TRACKING_MODE_FULL_HAND;
// 配置收的Tracking Mode
trackingMode = PXCMHandData.TrackingModeType.TRACKING_MODE_FULL_HAND;
handConfiguration.SetTrackingMode(trackingMode);
handConfiguration.EnableAllAlerts();
handConfiguration.EnableSegmentationImage(true);
bool isEnabled = handConfiguration.IsSegmentationImageEnabled();
handConfiguration.ApplyChanges();
int totalNumOfGestures = handConfiguration.QueryGesturesTotalNumber();
if (totalNumOfGestures > 0)
{
for (int i = 0; i < totalNumOfGestures; i++)
{
string gestureName = string.Empty;
if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) ==
pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Console.WriteLine(gestureName);
}
}
}
}
int frameCounter = 0;
int frameNumber = 0;
while (!m_form.Stopped)
{
string gestureName = "fist";
if (handConfiguration != null)
{
if (string.IsNullOrEmpty(gestureName) == false)
{
if (handConfiguration.IsGestureEnabled(gestureName) == false)
{
handConfiguration.DisableAllGestures();
handConfiguration.EnableGesture(gestureName, true);
handConfiguration.ApplyChanges();
}
}
else
{
handConfiguration.DisableAllGestures();
handConfiguration.ApplyChanges();
}
}
if (pp.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
break;
}
frameCounter++;
if (pp.IsConnected())
{
PXCMCapture.Sample sample;
sample = pp.QueryHandSample();
if (sample != null && sample.depth != null)
{
// frameNumber = liveCamera ? frameCounter : instance.captureManager.QueryFrameIndex();
frameNumber = frameCounter;
}
//bool b=(sample.ir==null);
//b = (sample.color== null);
//b = (sample.left == null);
//b = (sample.right == null);
DisplayPicture(sample.depth);
if (handData != null)
{
handData.Update();
SaveHandData(handData);
}
m_form.UpdatePic();
}
pp.ReleaseFrame();
}
if (handData != null)
{
handData.Dispose();
}
if (handConfiguration != null)
{
handConfiguration.Dispose();
}
pp.Close();
pp.Dispose();
}
private void ProcessingHandThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
hand = senseManager.QueryHand();
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
// Get number of tracked hands
nhands = handData.QueryNumberOfHands();
if (nhands > 0)
{
// Retrieve hand identifier
handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId);
// Retrieve hand data
handData.QueryHandDataById(handId, out ihand);
// Retrieve all hand joint data
for (int i = 0; i < nhands; i++)
{
for (int j = 0; j < 0x20; j++)
{
PXCMHandData.JointData jointData;
ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
nodes[i][j] = jointData;
}
}
// Get world coordinates for tip of middle finger on the first hand in camera range
handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x;
handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y;
handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z;
// Retrieve gesture data
if (handData.IsGestureFired("spreadfingers", out gestureData))
{
gesture = Gesture.FingerSpread;
}
else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
gesture = Gesture.Pinch;
}
else if (handData.IsGestureFired("wave", out gestureData))
{
gesture = Gesture.Wave;
}
}
else
{
gesture = Gesture.Undefined;
}
// Get alert status
for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++)
{
PXCMHandData.AlertData alertData;
if (handData.QueryFiredAlertData(i, out alertData) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; }
//Displaying last alert
switch (alertData.label)
{
case PXCMHandData.AlertType.ALERT_HAND_DETECTED:
detectionAlert = "Hand Detected";
detectionStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED:
detectionAlert = "Hand Not Detected";
detectionStatusOk = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED:
calibrationAlert = "Hand Calibrated";
calibrationStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED:
calibrationAlert = "Hand Not Calibrated";
calibrationStatusOk = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS:
bordersAlert = "Hand Inside Borders";
borderStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS:
bordersAlert = "Hand Out Of Borders";
borderStatusOk = false;
break;
}
}
UpdateUI();
if (handData != null) handData.Dispose();
}
senseManager.ReleaseFrame();
}
}
public void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
hand = senseManager.QueryHand();
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
// Get number of tracked hands
nhands = handData.QueryNumberOfHands();
if (nhands > 0)
{
// Retrieve hand identifier
handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId);
// Retrieve hand data
handData.QueryHandDataById(handId, out ihand);
PXCMHandData.BodySideType bodySideType = ihand.QueryBodySide();
if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
leftHand = true;
}
else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
leftHand = false;
}
// Retrieve all hand joint data
for (int i = 0; i < nhands; i++)
{
for (int j = 0; j < 0x20; j++)
{
PXCMHandData.JointData jointData;
ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
nodes[i][j] = jointData;
}
}
// Get world coordinates for tip of middle finger on the first hand in camera range
handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x;
handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y;
handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z;
swipehandTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.x;
swipehandTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.y;
swipehandTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.z;
//Console.Out.WriteLine("Before x={0}", swipehandTipX);
//Console.Out.WriteLine("Before speed={0}", nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].speed.x);
// Retrieve gesture data
if (handData.IsGestureFired("spreadfingers", out gestureData))
{
gesture = Gesture.FingerSpread;
}
else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
gesture = Gesture.Pinch;
}
else if (handData.IsGestureFired("wave", out gestureData))
{
gesture = Gesture.Wave;
}
else if (handData.IsGestureFired("swipe_left", out gestureData))
{
gesture = Gesture.SwipeLeft;
}
else if (handData.IsGestureFired("swipe_right", out gestureData))
{
gesture = Gesture.SwipeRight;
}
else if (handData.IsGestureFired("fist", out gestureData))
{
gesture = Gesture.Fist;
}
else if (handData.IsGestureFired("thumb_up", out gestureData))
{
gesture = Gesture.Thumb;
}
}
else
{
gesture = Gesture.Undefined;
}
UpdateUI();
if (handData != null)
{
handData.Dispose();
}
}
senseManager.ReleaseFrame();
}
}
/* Using PXCMSenseManager to handle data */
public void SimplePipeline()
{
form.UpdateInfo(String.Empty, Color.Black);
bool liveCamera = false;
bool flag = true;
PXCMSenseManager instance = null;
_disconnected = false;
instance = form.g_session.CreateSenseManager();
if (instance == null)
{
form.UpdateStatus("Failed creating SenseManager");
return;
}
if (form.GetRecordState())
{
instance.captureManager.SetFileName(form.GetFileName(), true);
PXCMCapture.DeviceInfo info;
if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info))
{
instance.captureManager.FilterByDeviceInfo(info);
}
}
else if (form.GetPlaybackState())
{
instance.captureManager.SetFileName(form.GetFileName(), false);
instance.captureManager.SetRealtime(false);
}
else
{
PXCMCapture.DeviceInfo info;
if (String.IsNullOrEmpty(form.GetCheckedDevice()))
{
form.UpdateStatus("Device Failure");
return;
}
if (form.Devices.TryGetValue(form.GetCheckedDevice(), out info))
{
instance.captureManager.FilterByDeviceInfo(info);
}
liveCamera = true;
}
/* Set Module */
pxcmStatus status = instance.EnableHand(form.GetCheckedModule());
PXCMHandModule handAnalysis = instance.QueryHand();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null)
{
form.UpdateStatus("Failed Loading Module");
return;
}
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler();
handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
PXCMHandData handData = handAnalysis.CreateOutput();
if (handConfiguration == null)
{
form.UpdateStatus("Failed Create Configuration");
return;
}
if (handData == null)
{
form.UpdateStatus("Failed Create Output");
return;
}
if (form.getInitGesturesFirstTime() == false)
{
int totalNumOfGestures = handConfiguration.QueryGesturesTotalNumber();
if (totalNumOfGestures > 0)
{
this.form.UpdateGesturesToList("", 0);
for (int i = 0; i < totalNumOfGestures; i++)
{
string gestureName = string.Empty;
if (handConfiguration.QueryGestureNameByIndex(i, out gestureName) ==
pxcmStatus.PXCM_STATUS_NO_ERROR)
{
this.form.UpdateGesturesToList(gestureName, i + 1);
}
}
form.setInitGesturesFirstTime(true);
form.UpdateGesturesListSize();
}
}
FPSTimer timer = new FPSTimer(form);
form.UpdateStatus("Init Started");
if (handAnalysis != null && instance.Init(handler) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.DeviceInfo dinfo;
PXCMCapture.Device device = instance.captureManager.device;
if (device != null)
{
pxcmStatus result = device.QueryDeviceInfo(out dinfo);
if (result == pxcmStatus.PXCM_STATUS_NO_ERROR && dinfo != null && dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM)
{
device.SetDepthConfidenceThreshold(1);
device.SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED);
device.SetIVCAMFilterOption(6);
}
_maxRange = device.QueryDepthSensorRange().max;
}
if (handConfiguration != null)
{
handConfiguration.EnableAllAlerts();
handConfiguration.EnableSegmentationImage(true);
handConfiguration.ApplyChanges();
handConfiguration.Update();
}
form.UpdateStatus("Streaming");
int frameCounter = 0;
int frameNumber = 0;
string nextPageGesture, previousPageGesture, firstPageGesture, endPageGesture;
HandsRecognition.Hand nextHand, previousHand, firstHand, endHand;
while (!form.stop)
{
form.GetHandType(out nextHand, out previousHand, out firstHand, out endHand);
form.GetGestureName(out nextPageGesture, out previousPageGesture, out firstPageGesture, out endPageGesture);
handConfiguration.DisableAllGestures();
if (string.IsNullOrEmpty(nextPageGesture) == false && handConfiguration.IsGestureEnabled(nextPageGesture) == false)
{
handConfiguration.EnableGesture(nextPageGesture, true);
NextPageGesture.Gesture = nextPageGesture;
NextPageGesture.handler = form.NextPage;
NextPageGesture.HandType = nextHand;
}
if (string.IsNullOrEmpty(previousPageGesture) == false && handConfiguration.IsGestureEnabled(previousPageGesture) == false)
{
handConfiguration.EnableGesture(previousPageGesture, true);
PreviousPageGesture.Gesture = previousPageGesture;
PreviousPageGesture.handler = form.PreviousPage;
PreviousPageGesture.HandType = previousHand;
}
if (string.IsNullOrEmpty(firstPageGesture) == false && handConfiguration.IsGestureEnabled(firstPageGesture) == false)
{
handConfiguration.EnableGesture(firstPageGesture, true);
FirstPageGesture.Gesture = firstPageGesture;
FirstPageGesture.handler = form.FirstPage;
FirstPageGesture.HandType = firstHand;
}
if (string.IsNullOrEmpty(endPageGesture) == false && handConfiguration.IsGestureEnabled(endPageGesture) == false)
{
handConfiguration.EnableGesture(endPageGesture, true);
EndPageGesture.Gesture = endPageGesture;
EndPageGesture.handler = form.EndPage;
EndPageGesture.HandType = endHand;
}
handConfiguration.ApplyChanges();
if (instance.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
break;
}
frameCounter++;
if (!DisplayDeviceConnection(!instance.IsConnected()))
{
if (handData != null)
{
handData.Update();
}
PXCMCapture.Sample sample = instance.QueryHandSample();
if (sample != null && sample.depth != null)
{
DisplayPicture(sample.depth, handData);
if (handData != null)
{
frameNumber = liveCamera ? frameCounter : instance.captureManager.QueryFrameIndex();
DisplayJoints(handData);
DisplayGesture(handData, frameNumber);
DisplayAlerts(handData, frameNumber);
}
form.UpdatePanel();
}
timer.Tick();
}
instance.ReleaseFrame();
}
// Clean Up
if (handData != null)
{
handData.Dispose();
}
if (handConfiguration != null)
{
handConfiguration.Dispose();
}
}
else
{
form.UpdateStatus("Init Failed");
flag = false;
}
foreach (PXCMImage pxcmImage in m_images)
{
pxcmImage.Dispose();
}
instance.Close();
instance.Dispose();
if (flag)
{
form.UpdateStatus("Stopped");
}
}
public Main.HandCoord[] GetJointCoordinates()
{
#region Catch exception errors
/* Wait until frame is available and SenseManager is ready */
/* Make sure SenseManager has an instance */
if (SenseManager == null)
{
return(null);
}
/* Wait until any frame data is available */
if (SenseManager.AcquireFrame(true) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return(null);
}
#endregion
#region Get joint coordinates
HandModule = SenseManager.QueryHand(); // Get hand tracking Module instance
HandData = HandModule.CreateOutput(); // Get hand tracking data
if (HandData != null)
{
HandData.Update(); // Update hand data to the most current output
/*Reset the detected hands to zero*/
for (int i = 0; i < NumOfHands; i++)
{
Coordinates[i].HandDetected = false;
}
/* Retrieve all joint data */
for (int i = 0; i < HandData.QueryNumberOfHands(); i++)
{
Coordinates[i].HandDetected = true;
PXCMHandData.IHand iHand;
Status = HandData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out iHand);
if (Status != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError(Status);
}
else
{
for (int j = 0; j < NumOfJoints; j++)
{
Status = iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out JointData[i][j]);
if (Status == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Coordinates[i].Coordinates[0, j] = JointData[i][j].positionWorld.x * -100;
Coordinates[i].Coordinates[1, j] = JointData[i][j].positionWorld.y * 100;
Coordinates[i].Coordinates[2, j] = JointData[i][j].positionWorld.z * 100;
Coordinates[i].Coordinates[3, j] = JointData[i][j].speed.x * -100;
Coordinates[i].Coordinates[4, j] = JointData[i][j].speed.y * -100;
Coordinates[i].Coordinates[5, j] = JointData[i][j].speed.z * 100;
}
}
}
}
}
SenseManager.ReleaseFrame(); // Prepare for a new frame
return(Coordinates);
#endregion
}
void Update()
{
//Dynamically Pause/Enable Modules
int numberOfEnabledModules = 0;
foreach (var option in _senseOptions)
{
if (option.RefCounter == 0 && option.Enabled)
{
if (option.ModuleCUID > 0)
{
SenseManager.PauseModule(option.ModuleCUID, true);
}
option.Enabled = false;
}
else if (option.RefCounter > 0 && !option.Enabled)
{
if (!option.Initialized)
{
OnDisable();
OnEnable();
Start();
}
if (option.ModuleCUID > 0)
{
SenseManager.PauseModule(option.ModuleCUID, false);
}
option.Enabled = true;
}
if (option.Enabled)
{
numberOfEnabledModules++;
}
}
//Update Speech commands if changed
if (_speechCommandsChanged)
{
UpdateSpeechCommands();
SpeechManager.Reset();
}
// Every frame update all the data
if (Initialized && numberOfEnabledModules > 0)
{
_sts = SenseManager.AcquireFrame(true, 100);
if (_sts == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled)
{
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.color != null)
{
ImageRgbOutput = _captureSample.color;
ImageRgbOutput.QueryInstance <PXCMAddRef>().AddRef();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled)
{
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.depth != null)
{
ImageDepthOutput = _captureSample.depth;
ImageDepthOutput.QueryInstance <PXCMAddRef>().AddRef();
if (PointCloud == null)
{
PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled)
{
if (PointCloud == null)
{
PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
_sts = _projection.QueryVertices(ImageDepthOutput, PointCloud);
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.UVMap).Enabled)
{
if (UvMap == null)
{
UvMap = new PXCMPointF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
_projection.QueryUVMap(ImageDepthOutput, UvMap);
}
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled)
{
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.ir != null)
{
ImageIROutput = _captureSample.ir;
ImageIROutput.QueryInstance <PXCMAddRef>().AddRef();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled)
{
FaceModuleOutput.Update();
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled)
{
HandDataOutput.Update();
}
_captureSample = null;
SenseManager.ReleaseFrame();
}
//Speech
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled)
{
SpeechManager.QueryRecognizedCommands(out SpeechOutput);
}
}
}
pxcmStatus newHandFrame(PXCMHandModule hand)
{
if (hand != null)
{
PXCMHandData handData = hand.CreateOutput();
handData.Update();
PXCMHandData.IHand iHandDataLeft = null, iHandDataRight = null;
PXCMHandData.JointData jointData = null;
PXCMImage image = null;
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out iHandDataLeft);
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out iHandDataRight);
if (handForm != null && !handForm.IsDisposed)
{
this.handForm.HandCount = handData.QueryNumberOfHands();
if (iHandDataLeft != null)
{
iHandDataLeft.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.LeftHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
if (iHandDataRight != null)
{
iHandDataRight.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.RightHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
}
if (iHandDataLeft != null)
{
if (jointData == null)
{
iHandDataLeft.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (iHandDataRight != null)
{
if (jointData == null)
{
iHandDataRight.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (jointData != null && canTrack.Checked)
{
Cursor.Position = new System.Drawing.Point(
(int)((640.0f - jointData.positionImage.x) * Screen.PrimaryScreen.Bounds.Width / 640.0f),
(int)(jointData.positionImage.y * Screen.PrimaryScreen.Bounds.Height / 480.0f));
PXCMHandData.GestureData gestureData = null;
if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
Program.DoMouseClick();
}
Console.WriteLine("Z Position: " + jointData.positionWorld.z);
}
handData.Dispose();
}
return(pxcmStatus.PXCM_STATUS_NO_ERROR);
}
// Update is called once per frame
void Update()
{
/* Make sure SenseManager Instance is valid */
if (sm == null)
{
return;
}
/* Wait until any frame data is available */
if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve hand tracking Module Instance */
handAnalyzer = sm.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput();
if (_handData != null)
{
_handData.Update();
/* Retrieve all joint Data */
bool[] someJointsDetected = { false, false };
for (int i = 0; i < _handData.QueryNumberOfHands(); i++)
{
PXCMHandData.IHand _iHand;
if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
for (int j = 0; j < MaxJoints; j++)
{
if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i, j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
jointData[i, j] = null;
}
else if (jointData[i, j].confidence == 100)
{
someJointsDetected[i] = true;
}
}
if (!handList.ContainsKey(_iHand.QueryUniqueId()))
{
handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide());
}
}
}
for (int i = 0; i < MaxHands; i++)
{
if (!someJointsDetected[i] && handWasTracked[i])
{
for (int j = 0; j < MaxJoints; j++)
{
Joint joint = myJoints[i, j];
if (joint == null)
{
continue;
}
joint.transform.position = joint.originalPosition;
joint.transform.rotation = joint.originalRotation;
foreach (Joint e in joint.extensions)
{
e.transform.position = e.originalPosition;
e.transform.rotation = e.originalRotation;
}
}
}
else
{
if (!handWasTracked[i])
{
if (myJoints[i, 0] == null || jointData[i, 0] == null || jointData[i, 0].confidence < 100)
{
break;
}
PXCMPoint3DF32 smoothedPoint = smoother3D[i, 0].SmoothValue(jointData[i, 0].positionWorld);
Vector3 targetPosition = new Vector3(-1 * smoothedPoint.x, smoothedPoint.y, smoothedPoint.z);
trackingOffset[i] = targetPosition - myJoints[i, 0].originalPosition;
}
/* Smooth the data and move the joints*/
MoveJoints(i);
}
handWasTracked[i] = someJointsDetected[i];
}
}
handAnalyzer.Dispose();
}
sm.ReleaseFrame();
RotateCam();
}
// Update is called once per frame
void Update()
{
// For testing only.
//handRenderer.queryLeftHand2DCoordinates ();
_outputData = SenseToolkitManager.Instance.HandDataOutput;
if (_outputData != null)
{
_outputData.Update();
/* Retrieve Hand Joints*/
joints = new PXCMHandData.JointData[2, PXCMHandData.NUMBER_OF_JOINTS];
for (int i = 0; i < _outputData.QueryNumberOfHands(); i++)
{
PXCMHandData.IHand _handData;
_outputData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _handData);
//Debug.Log(_handData.IsCalibrated().ToString());
//if(_handData!=null){
//else { Debug.Log ("No data"); continue; }
for (int j = 0; j < PXCMHandData.NUMBER_OF_JOINTS; j++)
{
_handData.QueryTrackedJoint((PXCMHandData.JointType)j, out joints[i, j]);
}
handIds[i] = _handData.QueryUniqueId();
bodySides[i] = _handData.QueryBodySide();
if (!_handData.IsCalibrated())
{
if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
isleftCalibrated = false;
}
else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
isrightCalibrated = false;
}
}
else
{
if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
{
isleftCalibrated = true;
}
else if (bodySides[i] == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
{
isrightCalibrated = true;
}
}
}
handRenderer.DisplaySmoothenedJoints(_outputData, joints, handIds, bodySides, _outputData.QueryNumberOfHands());
/* Retrieve Alert Data */
/*
* PXCMHandData.AlertData _alertData;
* for (int i = 0; i < _outputData.QueryFiredAlertsNumber(); i++)
* if (_outputData.QueryFiredAlertData(i, out _alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
* handRenderer.DisplayAlerts(_alertData);
*/
/* Retrieve Gesture Data */
PXCMHandData.GestureData _gestureData;
for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++)
{
if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
handRenderer.DisplayGestures(_gestureData);
}
}
if (_outputData.QueryNumberOfHands() == 0)
{
handRenderer.makeNull();
}
}
else
{
handRenderer.makeNull();
}
// For testing
//handRenderer.DisplayGest ();
displayHandGestures();
}
private void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop - MAIN PROCESSING LOOP
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (firstFrame == true)
{
firstFrame = false;
//pipeClient.SendMessage(CAMERA_CONNECTED_MESSAGE);
}
//Get sample from the sensemanager to convert to bitmap and show
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData = null;
// Get color/ir image data
if (cameraMode == "Color")
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
else if (cameraMode == "IR")
sample.ir.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
else if (cameraMode == "Depth")
;// -> broken! // sample.depth.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH, out colorData);
else
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
//convert it to bitmap
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Retrieve hand and face data AND EMOTION DATA
hand = senseManager.QueryHand();
face = senseManager.QueryFace();
emotion = senseManager.QueryEmotion();
//Process hand data
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
handWaving = handData.IsGestureFired("wave", out gestureData);
}
//Process face data
if (face != null)
{
// Retrieve the most recent processed data
faceData = face.CreateOutput();
faceData.Update();
numFacesDetected = faceData.QueryNumberOfDetectedFaces();
if (numFacesDetected > 0)
{
// for (Int32 i = 0; i < numFacesDetected; i++) --> MULTIPLE FACE DETECTION DISABLED, UNCOMMENT TO INCLUDE
// {
// PXCMFaceData.Face singleFace = faceData.QueryFaceByIndex(i); --> FOR MULTIPLE FACE DETECTION
//get all possible data from frame
PXCMFaceData.Face singleFaceData = faceData.QueryFaceByIndex(0); //only getting first face!
PXCMFaceData.ExpressionsData singleExprData = singleFaceData.QueryExpressions();
PXCMFaceData.DetectionData detectionData = singleFaceData.QueryDetection();
PXCMFaceData.LandmarksData landmarksData = singleFaceData.QueryLandmarks();
PXCMFaceData.PoseData poseData = singleFaceData.QueryPose();
//Work on face location data from detectionData
if (detectionData != null)
{
// vars are defined globally
detectionData.QueryBoundingRect(out boundingRect);
detectionData.QueryFaceAverageDepth(out averageDepth);
}
//Work on getting landmark data
if (landmarksData != null)
{
//var is defined globally
landmarksData.QueryPoints(out landmarkPoints);
}
//Work on getting euler angles for face pose data
if (poseData != null)
{
//var is defined globally
poseData.QueryPoseAngles(out eulerAngles);
poseData.QueryPoseQuaternion(out quaternionAngles);
}
//Do work on all face location data from singleExprData
if (singleExprData != null)
{
//get scores and intensities for right and left eye closing - 22 possible expressions --> put into hashtable
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
//this gets a list of enum names as strings
var enumNames = Enum.GetNames(typeof(PXCMFaceData.ExpressionsData.FaceExpression));
//for all enumnames, calculate the
for (int j = 0; j < enumNames.Length; j++)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult innerScore;
singleExprData.QueryExpression((PXCMFaceData.ExpressionsData.FaceExpression)(j), out innerScore);
//Console.WriteLine((PXCMFaceData.ExpressionsData.FaceExpression)(j));
exprTable[enumNames[j]] = innerScore.intensity;
}
//Attempt to write to file if there are any significant events
/* //check if everything is 0
bool significantEntry = false;
foreach (DictionaryEntry entry in exprTable)
{
if (Convert.ToInt32(entry.Value.ToString()) != 0)
{
significantEntry = true;
break;
}
}
if (significantEntry) */
writeSignificantToFile(exprTable, boundingRect, averageDepth, landmarkPoints, eulerAngles, quaternionAngles);
singleExprData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT, out score);
lEyeClosedIntensity = score.intensity;
singleExprData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT, out score);
rEyeClosedIntensity = score.intensity;
//eye closed logic -> will be reset in UI thread after some number of frames
if (lEyeClosedIntensity >= EYE_CLOSED_DETECT_THRESHOLD)
lEyeClosed = true;
if (rEyeClosedIntensity >= EYE_CLOSED_DETECT_THRESHOLD)
rEyeClosed = true;
}
// }
}
}
if (emotion != null)
{
int numFaces = emotion.QueryNumFaces();
for (int fid = 0; fid < numFaces; fid++)
{
//TODO - MULTIPLE FACE IMPLEMENTATION?
//retrieve all est data
PXCMEmotion.EmotionData[] arrData = new PXCMEmotion.EmotionData[10];
emotion.QueryAllEmotionData(fid, out arrData);
//find emotion with maximum evidence
int idx_outstanding_emotion = 0;
int max_evidence = arrData[0].evidence;
for (int k = 1; k < 7; k++)
{
if (arrData[k].evidence < max_evidence)
{
}
else
{
max_evidence = arrData[k].evidence;
idx_outstanding_emotion = k;
}
}
currentEmotion = arrData[idx_outstanding_emotion].eid;
//Console.WriteLine(currentEmotion.ToString());
emotionEvidence = max_evidence;
// Console.WriteLine(currentEmotion.ToString() + ":" + emotionEvidence.ToString());
}
}
// Update the user interface
UpdateUI(colorBitmap);
// Release the frame
if (handData != null) handData.Dispose();
// colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
senseManager.ReleaseFrame();
}
}
