private void Uninitialize()
{
if (senseManager != null)
{
senseManager.Dispose();
senseManager = null;
}
if (projection != null)
{
projection.Dispose();
projection = null;
}
if (handData != null)
{
handData.Dispose();
handData = null;
}
if (handAnalyzer != null)
{
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
private int weightsNum = 15; //smoothing factor
// Use this for initialization
void Start () {
hands = new HandsModel ();
MaxHands = hands.MaxHands;
MaxJoints = hands.MaxJoints;
handList = new Hashtable ();
senseManager = PXCMSenseManager.CreateInstance ();
if (senseManager == null)
Debug.LogError ("SenseManager Initialization Failed");
/* Enable hand tracking and retrieve an hand module instance to configure */
status = senseManager.EnableHand ();
handAnalyzer = senseManager.QueryHand ();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError ("PXCSenseManager.EnableHand: " + status);
/* Initialize the execution pipeline */
status = senseManager.Init ();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError ("PXCSenseManager.Init: " + status);
/* Retrieve the the DataSmoothing instance */
senseManager.QuerySession ().CreateImpl<PXCMDataSmoothing> (out dataSmoothing);
smoother3D = new PXCMDataSmoothing.Smoother3D[MaxHands][];
/* Configure a hand - Enable Gestures and Alerts */
PXCMHandConfiguration hcfg = handAnalyzer.CreateActiveConfiguration ();
hcfg.EnableAllGestures ();
hcfg.EnableAlert (PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED);
hcfg.ApplyChanges ();
hcfg.Dispose ();
InitObject ();
}
/// <summary> 終了処理 </summary>
private void Uninitialize()
{
if (senseManager != null)
{
senseManager.Dispose();
senseManager = null;
}
if (projection != null)
{
projection.Dispose();
projection = null;
}
if (handData != null)
{
handData.Dispose();
handData = null;
}
if (handAnalyzer != null)
{
handAnalyzer.Dispose();
handAnalyzer = null;
}
//handConfig.UnsubscribeGesture(OnFiredGesture);
//handConfig.Dispose();
}
private void Uninitialize()
{
if (senseManager != null)
{
senseManager.Dispose();
senseManager = null;
}
if (handConfig != null)
{
handConfig.Dispose();
handConfig = null;
}
if (handData != null)
{
handData.Dispose();
handData = null;
}
if (handAnalyzer != null)
{
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
public Camera()
{
this.action = new Action();
this.rockCounter = 0;
directionTimer = new System.Timers.Timer(1000);
directionTimer.Elapsed += directionTimer_Elapsed;
pinchTimer = new System.Timers.Timer(500);
pinchTimer.Elapsed += pinchTimer_Elapsed;
rockTimer = new System.Timers.Timer(300);
rockTimer.Elapsed += rockTimer_Elapsed;
// Create the manager
this._session = PXCMSession.CreateInstance();
this._mngr = this._session.CreateSenseManager();
// streammmm
PXCMVideoModule.DataDesc desc = new PXCMVideoModule.DataDesc();
desc.deviceInfo.streams = PXCMCapture.StreamType.STREAM_TYPE_COLOR | PXCMCapture.StreamType.STREAM_TYPE_DEPTH;
this._mngr.EnableStreams(desc);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, Camera.WIDTH, Camera.HEIGHT, 30);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, Camera.WIDTH, Camera.HEIGHT, 30);
// Hands
this._mngr.EnableHand();
this._hand = this._mngr.QueryHand();
this._handData = this._hand.CreateOutput();
// Hands config
PXCMHandConfiguration conf = this._hand.CreateActiveConfiguration();
conf.EnableGesture("two_fingers_pinch_open", true);
// Subscribe hands alerts
conf.EnableAllAlerts();
//conf.SubscribeAlert(this.onFiredAlert);
conf.EnableTrackedJoints(true);
// and the private one for debug
//conf.SubscribeGesture(this.onFiredGesture);
// Apply it all
conf.ApplyChanges();
// Set events
this._handler = new PXCMSenseManager.Handler();
this._handler.onModuleProcessedFrame = this.onModuleProcessedFrame;
this._mngr.Init(this._handler);
}
// Initialise all the things
public override void Init(PXCMSenseManager sManager)
{
senseManager = sManager;
senseManager.EnableHand();
module = senseManager.QueryHand();
PXCMHandConfiguration config = module.CreateActiveConfiguration();
config.SetTrackingMode(PXCMHandData.TrackingModeType.TRACKING_MODE_FULL_HAND);
config.ApplyChanges();
config.Update();
}
// Update is called once per frame
void Update()
{
/* Make sure PXCMSenseManager Instance is Initialized */
if (SenseToolkitManager.Instance.SenseManager == null)
{
return;
}
/* Wait until any frame data is available true(aligned) false(unaligned) */
if (SenseToolkitManager.Instance.SenseManager.AcquireFrame(false, 0) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve am instance of hand tracking Module */
handAnalyzer = SenseToolkitManager.Instance.SenseManager.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve an instance of hand tracking Data */
PXCMHandData _outputData = handAnalyzer.CreateOutput();
if (_outputData != null)
{
_outputData.Update();
//Retrieve joint data, gesture recognition data and alert notification data
//refer to next section
//AcquireFrame
/* Retrieve Gesture Data */
PXCMHandData.GestureData _gestureData;
for (int i = 0; i < _outputData.QueryFiredGesturesNumber(); i++)
{
if (_outputData.QueryFiredGestureData(i, out _gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
//Display the gestures: explained in rendering the frame section
if ((_gestureData.name == "spreadfingers") || (_gestureData.name == "tap") ||
(_gestureData.name == "swipe"))
{
CursorController.isHandClicked = false;
}
else if ((_gestureData.name == "fist") || (_gestureData.name == "full_pinch") ||
(_gestureData.name == "thumb_down") || (_gestureData.name == "thumb_up") ||
(_gestureData.name == "two_fingers_pinch_open") || (_gestureData.name == "v_sign"))
{
CursorController.isHandClicked = true;
}
else if (_gestureData.name == "wave")
{
transform.position = new Vector3(0, 0, 0);
}
}
}
}
}
/* Realease the frame to process the next frame */
SenseToolkitManager.Instance.SenseManager.ReleaseFrame();
}
// Use this for initialization
void Start()
{
// Creates an instance of the sense manager to be called later
session = PXCMSenseManager.CreateInstance();
//Output an error if there is no instance of the sense manager
if (session == null)
{
Debug.LogError("SenseManager Init Failed!");
}
// Enables hand tracking
sts = session.EnableHand();
handAnalyzer = session.QueryHand();
sts2 = session.EnableFace();
faceAnalyzer = session.QueryFace();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError("PXCSenseManager.EnableHand: " + sts);
if (sts2 != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError("PXCSenseManager.EnableFace: " + sts2);
// Creates the session
sts = session.Init();
sts2 = session.Init();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError("PXCSenseManager.Init: " + sts);
if (sts2 != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError("PXCSenseManager.Init: " + sts2);
// Creates a hand config for future data
PXCMHandConfiguration handconfig = handAnalyzer.CreateActiveConfiguration();
PXCMFaceConfiguration faceconfig = faceAnalyzer.CreateActiveConfiguration();
//If there is handconfig instance
if (handconfig != null)
{
handconfig.EnableAllAlerts();
handconfig.ApplyChanges();
handconfig.Dispose();
}
if (faceconfig != null)
{
faceconfig.EnableAllAlerts();
faceconfig.ApplyChanges();
faceconfig.Dispose();
}
}
private void StartRealSense()
{
bool useHead = bool.Parse(ConfigurationManager.AppSettings["UseHead"]);
// Instantiate and initialize the SenseManager
senseManager = PXCMSenseManager.CreateInstance();
// Configure the Hand Module
if (useHead)
{
senseManager.EnableFace();
face = senseManager.QueryFace();
faceConfig = face.CreateActiveConfiguration();
faceConfig.detection.isEnabled = true;
faceConfig.QueryExpressions().Enable();
faceConfig.ApplyChanges();
}
else
{
// Enable cursor tracking
senseManager.EnableHand();
// Get an instance of the hand cursor module
hand = senseManager.QueryHand();
// Get an instance of the cursor configuration
var cursorConfig = hand.CreateActiveConfiguration();
// Make configuration changes and apply them
cursorConfig.DisableAllAlerts();
cursorConfig.EnableTrackedJoints(true);
cursorConfig.EnableStabilizer(true);
cursorConfig.ApplyChanges();
}
senseManager.Init();
// Create an instance of PXCMSmoother
senseManager.session.CreateImpl <PXCMSmoother>(out smoother);
smoother2D = smoother.Create2DQuadratic(.5F);
smoother2D2 = smoother.Create2DQuadratic(1);
// Start the worker thread
processingThread = new Thread(new ThreadStart(ProcessingThread));
processingThread.Start();
}
public HandsRecognition(int mod)
{
try
{
session = PXCMSession.CreateInstance();
mode = mod;
if (mod == 0)
{
rightcp = new checkPiano();
leftcp = new checkPiano();
}
else if (mod == 1)
{
rightcp = new checkDrum();
leftcp = new checkDrum();
}
_disconnected = false;
instance = session.CreateSenseManager();
if (instance == null)
{
MessageBox.Show("Failed creating SenseManager", "OnAlert");
return;
}
/* Set Module */
pxcmStatus status = instance.EnableHand();//form.GetCheckedModule());
handAnalysis = instance.QueryHand();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || handAnalysis == null)
{
MessageBox.Show("Failed Loading Module", "OnAlert");
return;
}
handler = new PXCMSenseManager.Handler();
handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
handData = handAnalysis.CreateOutput();
}
catch
{
MessageBox.Show("Init Failed.");
Environment.Exit(0);
}
}
void Start()
{
// Initialise a PXCMSenseManager instance
psm = PXCMSenseManager.CreateInstance();
if (psm == null)
{
Debug.LogError("SenseManager Init Failed");
return;
}
// Enable the depth and colour streams
psm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 640, 480);
psm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480);
// Enable hand analysis
pxcmStatus sts = psm.EnableHand();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("SenseManager Hand Init Failed");
OnDisable();
return;
}
handModule = psm.QueryHand();
// Initialise the execution pipeline
sts = psm.Init();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("SenseManager Pipeline Init Failed");
OnDisable();
return;
}
handData = handModule.CreateOutput();
handConfig = handModule.CreateActiveConfiguration();
handConfig.EnableAllGestures();
handConfig.ApplyChanges();
foreach (CapsuleCollider capsule in GetComponentsInChildren <CapsuleCollider>())
{
hands.Add(capsule.gameObject);
}
mainCamera = GetComponentInChildren <Camera>();
}
public override void Work(Graphics g)
{
// Retrieve gesture data
hand = senseManager.QueryHand();
face = senseManager.QueryFace(); //
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
handWaving = handData.IsGestureFired("wave", out gestureData);
}
if (face != null)
{
faceData = face.CreateOutput();
faceData.Update();
//surching faces
Int32 nfaces = faceData.QueryNumberOfDetectedFaces();
for (Int32 i = 0; i < nfaces; i++)
{
// Retrieve the data instance
PXCMFaceData.Face faceI = faceData.QueryFaceByIndex(i);
PXCMFaceData.ExpressionsData edata = faceI.QueryExpressions();
if (edata != null)
{
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out score);
if (score.intensity >= 25)
{
smiling = true;
}
else
{
smiling = false;
}
Console.WriteLine(i + ": " + score.intensity);
}
}
faceData.Dispose();
}
}
// 手の検出の初期化
private void InitializeHandTracking()
{
// 手の検出器を取得する
handAnalyzer = senseManager.QueryHand();
if (handAnalyzer == null)
{
throw new Exception("手の検出器の取得に失敗しました");
}
// 手のデータを作成する
handData = handAnalyzer.CreateOutput();
if (handData == null)
{
throw new Exception("手の検出器の作成に失敗しました");
}
// RealSense カメラであれば、プロパティを設定する
var device = senseManager.QueryCaptureManager().QueryDevice();
PXCMCapture.DeviceInfo dinfo;
device.QueryDeviceInfo(out dinfo);
if (dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM)
{
device.SetDepthConfidenceThreshold(1);
//device.SetMirrorMode( PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED );
device.SetIVCAMFilterOption(6);
}
// 手の検出の設定
handConfig = handAnalyzer.CreateActiveConfiguration();
// 登録されているジェスチャーを列挙する
var num = handConfig.QueryGesturesTotalNumber();
for (int i = 0; i < num; i++)
{
string gestureName;
var sts = handConfig.QueryGestureNameByIndex(i, out gestureName);
if (sts == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
ComboGesture.Items.Add(gestureName);
}
}
handConfig.ApplyChanges();
handConfig.Update();
}
public Camera(params PXCMHandConfiguration.OnFiredGestureDelegate[] dlgts)
{
// Create the manager
this._session = PXCMSession.CreateInstance();
this._mngr = this._session.CreateSenseManager();
// streammmm
PXCMVideoModule.DataDesc desc = new PXCMVideoModule.DataDesc();
desc.deviceInfo.streams = PXCMCapture.StreamType.STREAM_TYPE_COLOR | PXCMCapture.StreamType.STREAM_TYPE_DEPTH;
this._mngr.EnableStreams(desc);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, Camera.WIDTH, Camera.HEIGHT, 30);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, Camera.WIDTH, Camera.HEIGHT, 30);
// Hands
this._mngr.EnableHand();
this._hand = this._mngr.QueryHand();
this._handData = this._hand.CreateOutput();
// Hands config
PXCMHandConfiguration conf = this._hand.CreateActiveConfiguration();
conf.EnableGesture("spreadfingers", false);
conf.EnableGesture("thumb_up", false);
// Subscribe hands alerts
conf.EnableAllAlerts();
conf.SubscribeAlert(this.onFiredAlert);
// Subscribe all gestures
foreach (PXCMHandConfiguration.OnFiredGestureDelegate subscriber in dlgts)
{
conf.SubscribeGesture(subscriber);
}
// and the private one for debug
conf.SubscribeGesture(this.onFiredGesture);
// Apply it all
conf.ApplyChanges();
// Set events
this._handler = new PXCMSenseManager.Handler();
this._handler.onModuleProcessedFrame = this.onModuleProcessedFrame;
this._mngr.Init(this._handler);
}
private void ConfigureHandModule()
{
timerCount = Stopwatch.StartNew();
hand = senseManager.QueryHand();
handConfig = hand.CreateActiveConfiguration();
handConfig.EnableGesture("spreadfingers");
handConfig.EnableGesture("two_fingers_pinch_open");
handConfig.EnableGesture("wave");
handConfig.EnableGesture("swipe");
handConfig.EnableGesture("swipe_left");
handConfig.EnableGesture("swipe_right");
handConfig.EnableGesture("fist");
handConfig.EnableGesture("thumb_up");
handConfig.EnableAllAlerts();
handConfig.ApplyChanges();
}
public Camera(params PXCMHandConfiguration.OnFiredGestureDelegate[] dlgts)
{
// Create the manager
this._session = PXCMSession.CreateInstance();
this._mngr = this._session.CreateSenseManager();
// streammmm
PXCMVideoModule.DataDesc desc = new PXCMVideoModule.DataDesc();
desc.deviceInfo.streams = PXCMCapture.StreamType.STREAM_TYPE_COLOR | PXCMCapture.StreamType.STREAM_TYPE_DEPTH;
this._mngr.EnableStreams(desc);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, Camera.WIDTH, Camera.HEIGHT, 30);
//this._mngr.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, Camera.WIDTH, Camera.HEIGHT, 30);
// Hands
this._mngr.EnableHand();
this._hand = this._mngr.QueryHand();
this._handData = this._hand.CreateOutput();
// Hands config
PXCMHandConfiguration conf = this._hand.CreateActiveConfiguration();
conf.EnableGesture("spreadfingers", false);
conf.EnableGesture("thumb_up", false);
// Subscribe hands alerts
conf.EnableAllAlerts();
conf.SubscribeAlert(this.onFiredAlert);
// Subscribe all gestures
foreach (PXCMHandConfiguration.OnFiredGestureDelegate subscriber in dlgts)
{
conf.SubscribeGesture(subscriber);
}
// and the private one for debug
conf.SubscribeGesture(this.onFiredGesture);
// Apply it all
conf.ApplyChanges();
// Set events
this._handler = new PXCMSenseManager.Handler();
this._handler.onModuleProcessedFrame = this.onModuleProcessedFrame;
this._mngr.Init(this._handler);
}
private Hashtable handList; //keep track of bodyside and hands for GUItext
// Use this for initialization
void Start()
{
handList = new Hashtable();
/* Initialize a PXCMSenseManager instance */
sm = PXCMSenseManager.CreateInstance();
if (sm == null)
{
Debug.LogError("SenseManager Initialization Failed");
}
/* Enable hand tracking and retrieve an hand module instance to configure */
sts = sm.EnableHand();
handAnalyzer = sm.QueryHand();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCSenseManager.EnableHand: " + sts);
}
/* Initialize the execution pipeline */
sts = sm.Init();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCSenseManager.Init: " + sts);
}
/* Retrieve the the DataSmoothing instance */
sm.QuerySession().CreateImpl <PXCMSmoother>(out smoother);
/* Create a 3D Weighted algorithm */
smoother3D = new PXCMSmoother.Smoother3D[MaxHands][];
/* Configure a hand - Enable Gestures and Alerts */
PXCMHandConfiguration hcfg = handAnalyzer.CreateActiveConfiguration();
if (hcfg != null)
{
hcfg.EnableAllGestures();
hcfg.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED);
hcfg.ApplyChanges();
hcfg.Dispose();
}
InitializeGameobjects();
}
public override void Init(PXCMSenseManager sManager)
{
this.senseManager = sManager;
hand = senseManager.QueryHand();
face = senseManager.QueryFace();
handConfig = hand.CreateActiveConfiguration();
handConfig.EnableGesture("wave");
handConfig.EnableAllAlerts();
handConfig.ApplyChanges();
faceConfic = face.CreateActiveConfiguration();
faceConfic.QueryExpressions();
faceConfic.EnableAllAlerts();
faceConfic.ApplyChanges();
PXCMFaceConfiguration.ExpressionsConfiguration expc = faceConfic.QueryExpressions();
expc.Enable();
expc.EnableAllExpressions();
faceConfic.ApplyChanges();
Console.WriteLine("init smile done");
}
void updateThread()
{
while (true)
{
if (manager.AcquireFrame(true) < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
break;
}
PXCMHandModule hand = manager.QueryHand();
PXCMCapture.Sample sample = manager.QuerySample();
this.newColorFrame(0, sample);
this.newDepthFrame(0, sample);
this.newIRFrame(0, sample);
this.newHandFrame(hand);
manager.ReleaseFrame();
}
}
/// <summary> 手の検出の初期化 </summary>
private void InitializeHandTracking()
{
// 手の検出器を取得する
handAnalyzer = senseManager.QueryHand();
if (handAnalyzer == null)
{
throw new Exception("手の検出器の取得に失敗しました");
}
// 手のデータを作成する
handData = handAnalyzer.CreateOutput();
if (handData == null)
{
throw new Exception("手の検出器の作成に失敗しました");
}
// RealSense カメラであれば、プロパティを設定する
var device = senseManager.QueryCaptureManager().QueryDevice();
PXCMCapture.DeviceInfo dinfo;
device.QueryDeviceInfo(out dinfo);
if (dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM)
{
device.SetDepthConfidenceThreshold(1);
//device.SetMirrorMode( PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED );
device.SetIVCAMFilterOption(6);
}
// 手の検出の設定
config = handAnalyzer.CreateActiveConfiguration();
//config.EnableSegmentationImage(true);
config.EnableJointSpeed(PXCMHandData.JointType.JOINT_MIDDLE_TIP, PXCMHandData.JointSpeedType.JOINT_SPEED_AVERAGE, 100);
//config.EnableGesture("v_sign");
config.EnableGesture("thumb_up");
config.EnableGesture("thumb_down");
//config.EnableGesture("tap");
//config.EnableGesture("fist");
config.SubscribeGesture(OnFiredGesture);
config.ApplyChanges();
config.Update();
gestureTimer.Start();
}
private void Uninitialize()
{
// リソースを解放する
if (senseManager != null)
{
senseManager.Dispose();
senseManager = null;
}
if (handData != null)
{
handData.Dispose();
handData = null;
}
if (handAnalyzer != null)
{
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
public MainWindow()
{
InitializeComponent();
// Instantiate and initialize the SenseManager
_senseManager = PXCMSenseManager.CreateInstance();
_senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 30);
_senseManager.EnableHand();
_senseManager.Init();
// Configure the Hand Module
_hand = _senseManager.QueryHand();
_handConfig = _hand.CreateActiveConfiguration();
_handConfig.EnableGesture("wave");
_handConfig.EnableAllAlerts();
_handConfig.ApplyChanges();
// Start the worker thread
_processingThread = new Thread(new ThreadStart(ProcessingThread));
_processingThread.Start();
}
public override void Listen()
{
// attach the controller to the PXCM sensor
_senseManager = Session.CreateSenseManager();
_senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 30);
_senseManager.EnableHand();
_handModule = _senseManager.QueryHand();
_handData = _handModule.CreateOutput();
_handConfiguration = _handModule.CreateActiveConfiguration();
_handConfiguration.SubscribeGesture(_handGestureHandler);
_handConfiguration.SubscribeAlert(_handAlertHandler);
_handConfiguration.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_TRACKED);
_handConfiguration.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_CALIBRATED);
_handConfiguration.EnableGesture("full_pinch");
_handConfiguration.EnableGesture("thumb_up");
_handConfiguration.ApplyChanges();
_senseManager.Init(_handler);
sensorActive = true;
_senseManager.StreamFrames(true);
_senseManager.Close();
}
void OnDisable()
{
/* Dispose hand data instance*/
if (hand_data != null)
{
hand_data.Dispose();
hand_data = null;
}
/* Dispose hand module instance*/
if (hand != null)
{
hand.Dispose();
hand = null;
}
/* Dispose sense manager instance*/
if (sm != null)
{
sm.Dispose();
sm = null;
}
}
// Use this for initialization
void Start()
{
// Set up the reference to the aeroplane controller.
m_Aeroplane = GetComponent <AeroplaneController>();
/* Initialize a PXCMSenseManager instance */
sm = PXCMSenseManager.CreateInstance();
if (sm != null)
{
/* Enable hand tracking and configure the hand module */
pxcmStatus sts = sm.EnableHand();
if (sts == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
/*init hand data structure*/
handData = new PXCMHandData.IHand[2];
/* Hand module interface instance */
hand = sm.QueryHand();
/* Hand data interface instance */
hand_data = hand.CreateOutput();
// Create hand configuration instance and configure
hcfg = hand.CreateActiveConfiguration();
hcfg.EnableAllAlerts();
hcfg.SubscribeAlert(OnFiredAlert);
hcfg.EnableNormalizedJoints(true);
hcfg.ApplyChanges();
hcfg.Dispose();
/* Initialize the execution pipeline */
if (sm.Init() != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
OnDisable();
}
}
}
}
public Model()
{
width = 640;
height = 480;
framerate = 30;
senseManager = PXCMSenseManager.CreateInstance();
senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, width, height, framerate);
// Enable Face detection
senseManager.EnableFace();
senseManager.EnableHand();
senseManager.Init();
face = senseManager.QueryFace();
faceConfig = face.CreateActiveConfiguration();
faceConfig.SetTrackingMode(PXCMFaceConfiguration.TrackingModeType.FACE_MODE_COLOR);
faceConfig.detection.isEnabled = true;
faceConfig.QueryExpressions();
PXCMFaceConfiguration.ExpressionsConfiguration expc = faceConfig.QueryExpressions();
expc.Enable();
expc.EnableAllExpressions();
faceConfig.ApplyChanges();
faceConfig.Update();
//faceData = face.CreateOutput();
//faceData.Update();
hand = senseManager.QueryHand();
PXCMHandConfiguration config = hand.CreateActiveConfiguration();
config.SetTrackingMode(PXCMHandData.TrackingModeType.TRACKING_MODE_FULL_HAND);
config.ApplyChanges();
config.Update();
//handData = hand.CreateOutput();
//handData.Update();
modules = new List <RSModule>();
}
// 手の検出の初期化
private void InitializeHandTracking()
{
// 手の検出器を取得する
handAnalyzer = senseManager.QueryHand();
if (handAnalyzer == null)
{
throw new Exception("手の検出器の取得に失敗しました");
}
// 手のデータを作成する
handData = handAnalyzer.CreateOutput();
if (handData == null)
{
throw new Exception("手の検出器の作成に失敗しました");
}
// RealSense カメラであれば、プロパティを設定する
var device = senseManager.QueryCaptureManager().QueryDevice();
PXCMCapture.DeviceInfo dinfo;
device.QueryDeviceInfo(out dinfo);
if (dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM)
{
device.SetDepthConfidenceThreshold(1);
//device.SetMirrorMode( PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED );
device.SetIVCAMFilterOption(6);
}
// 手の検出の設定
var config = handAnalyzer.CreateActiveConfiguration();
config.SetTrackingMode(PXCMHandData.TrackingModeType.TRACKING_MODE_EXTREMITIES);
config.EnableSegmentationImage(true);
config.ApplyChanges();
config.Update();
}
private void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
// Get color image data
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Retrieve gesture data
hand = senseManager.QueryHand();
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
handWaving = handData.IsGestureFired("v_sign", out gestureData);
}
// Update the user interface
UpdateUI(colorBitmap);
// Release the frame
if (handData != null)
{
handData.Dispose();
}
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
senseManager.ReleaseFrame();
}
}
void InitializeHandModule()
{
/* Enable the hand tracking module*/
sts = SenseToolkitManager.Instance.SenseManager.EnableHand();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCSenseManager.EnableHand: " + sts);
}
/* Retrieve an instance of hand to configure */
handAnalyzer = SenseToolkitManager.Instance.SenseManager.QueryHand();
if (handAnalyzer == null)
{
Debug.LogError("PXCSenseManager.QueryHand");
}
/* Initialize the execution pipeline */
sts = SenseToolkitManager.Instance.SenseManager.Init();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCMSenseManager.Init Failed");
OnDisable();
// Clean-up
return;
}
}
protected override void Uninitialize()
{
this.initialized = false;
if (this.projection != null)
{
this.projection.Dispose();
this.projection = null;
}
if (this.handData != null)
{
this.handData.Dispose();
this.handData = null;
}
if (this.handAnalyzer != null)
{
this.handAnalyzer.Dispose();
this.handAnalyzer = null;
}
base.Uninitialize();
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
_sts = SenseManager.EnableBlob();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) ||
IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
SenseManager.Enable3DSeg();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
//Set different configurations:
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
if (faceModule == null)
{
throw new UnityException("QueryFace returned null");
}
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15))
{
Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1");
NumberOfDetectedFaces = 1;
}
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
if (expressionConfig == null)
{
throw new UnityException("QueryExpressions returned null");
}
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
if (handAnalysis == null)
{
throw new UnityException("QueryHand returned null");
}
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableStabilizer(true);
handConfiguration.DisableAllAlerts();
handConfiguration.EnableSegmentationImage(false);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
// Blob
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
PXCMBlobModule blobAnalysis = SenseManager.QueryBlob();
if (blobAnalysis == null)
{
throw new UnityException("QueryBlob returned null");
}
PXCMBlobConfiguration blobConfiguration = blobAnalysis.CreateActiveConfiguration();
if (blobConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
blobConfiguration.Update();
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.EnableStabilizer(true);
blobConfiguration.SetMaxDistance(50 * 10);
blobConfiguration.ApplyChanges();
blobConfiguration.Dispose();
BlobDataOutput = blobAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection & blob extractor
if (Projection == null)
{
Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
/* GZ
* if (BlobExtractor == null)
* {
* SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor);
* }*/
// Set initialization flag
Initialized = true;
}
private void releaseInstances()
{
if (handModule != null)
{
handModule.Dispose();
handModule = null;
}
if (touchController != null)
{
touchController.Dispose();
touchController = null;
}
if (manager != null)
{
manager.Close();
manager.Dispose();
manager = null;
}
started = false;
}
public void InitializeGestureRecognition(List<String> gestures = null)
{
manager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 640, 480, 30);
var s = manager.EnableHand();
this.handAnalyzer = manager.QueryHand();
var re = manager.Init();
this.outputData = handAnalyzer.CreateOutput();
this.config = handAnalyzer.CreateActiveConfiguration();
if (gestures == null)
this.config.EnableAllGestures();
else
foreach (var g in gestures) this.config.EnableGesture(g);
this.config.ApplyChanges();
if (re != pxcmStatus.PXCM_STATUS_NO_ERROR)
throw new RealSenseException("error: " + re, re);
this.isGestureRecognitionEnabled = true;
}
// Update is called once per frame
void Update()
{
if (session == null)
return;
// For accessing hand data
handAnalyzer = session.QueryHand();
faceAnalyzer = session.QueryFace();
if (handAnalyzer != null)
{
PXCMHandData handData = handAnalyzer.CreateOutput();
if (handData != null)
{
handData.Update();
PXCMHandData.IHand IHAND; // Ihand instance for accessing future data
// Int32 IhandData; // for QueryOpenness Value
// PXCMPoint3DF32 location; // Stores hand tracking position
//Fills IHAND with information to later be grabbed and used for tracking + openness
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, 0, out IHAND);
// If there is data in Ihand
if (IHAND != null)
{
// Debug.DrawLine(transform.position, hit.point, Color.red);
// Inits hand tracking from the center of the hand.
// location = IHAND.QueryMassCenterWorld();
// if (mCurrentDart != null)
// {
// Vector3 locationUnity = new Vector3(location.x, location.y, location.z);
// mCurrentDart.transform.localPosition = locationUnity * RSScale;
// }
}
}
handAnalyzer.Dispose();
session.ReleaseFrame();
}
if (faceAnalyzer != null)
{
PXCMFaceData facedata = faceAnalyzer.CreateOutput();
if (facedata != null)
{
Int32 nfaces = facedata.QueryNumberOfDetectedFaces();
for (Int32 i = 0; i < nfaces; i++)
{
// Retrieve the face landmark data instance
PXCMFaceData.Face face = facedata.QueryFaceByIndex(i);
PXCMFaceData.PoseData pdata = face.QueryPose();
// retrieve the pose information
PXCMFaceData.PoseEulerAngles angles;
pdata.QueryPoseAngles(out angles);
Debug.Log("Eular Angles yaw : " + angles.yaw);
Debug.Log("Eular Angles pitch: " + angles.pitch);
Debug.Log("Eular Angles Roll: " + angles.roll);
angles.pitch = gameObject.transform.rotation.z;
angles.yaw = gameObject.transform.rotation.y;
}
// device is a PXCMCapture.Device instance
}
}
}
private void Uninitialize()
{
// リソースを解放する
if ( senseManager != null ) {
senseManager.Dispose();
senseManager = null;
}
if ( handData != null ) {
handData.Dispose();
handData = null;
}
if ( handAnalyzer != null ) {
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
private void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop - MAIN PROCESSING LOOP
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (firstFrame == true)
{
firstFrame = false;
//pipeClient.SendMessage(CAMERA_CONNECTED_MESSAGE);
}
//Get sample from the sensemanager to convert to bitmap and show
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData = null;
// Get color/ir image data
if (cameraMode == "Color")
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
else if (cameraMode == "IR")
sample.ir.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
else if (cameraMode == "Depth")
;// -> broken! // sample.depth.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_DEPTH, out colorData);
else
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
//convert it to bitmap
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Retrieve hand and face data AND EMOTION DATA
hand = senseManager.QueryHand();
face = senseManager.QueryFace();
emotion = senseManager.QueryEmotion();
//Process hand data
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
handWaving = handData.IsGestureFired("wave", out gestureData);
}
//Process face data
if (face != null)
{
// Retrieve the most recent processed data
faceData = face.CreateOutput();
faceData.Update();
numFacesDetected = faceData.QueryNumberOfDetectedFaces();
if (numFacesDetected > 0)
{
// for (Int32 i = 0; i < numFacesDetected; i++) --> MULTIPLE FACE DETECTION DISABLED, UNCOMMENT TO INCLUDE
// {
// PXCMFaceData.Face singleFace = faceData.QueryFaceByIndex(i); --> FOR MULTIPLE FACE DETECTION
//get all possible data from frame
PXCMFaceData.Face singleFaceData = faceData.QueryFaceByIndex(0); //only getting first face!
PXCMFaceData.ExpressionsData singleExprData = singleFaceData.QueryExpressions();
PXCMFaceData.DetectionData detectionData = singleFaceData.QueryDetection();
PXCMFaceData.LandmarksData landmarksData = singleFaceData.QueryLandmarks();
PXCMFaceData.PoseData poseData = singleFaceData.QueryPose();
//Work on face location data from detectionData
if (detectionData != null)
{
// vars are defined globally
detectionData.QueryBoundingRect(out boundingRect);
detectionData.QueryFaceAverageDepth(out averageDepth);
}
//Work on getting landmark data
if (landmarksData != null)
{
//var is defined globally
landmarksData.QueryPoints(out landmarkPoints);
}
//Work on getting euler angles for face pose data
if (poseData != null)
{
//var is defined globally
poseData.QueryPoseAngles(out eulerAngles);
poseData.QueryPoseQuaternion(out quaternionAngles);
}
//Do work on all face location data from singleExprData
if (singleExprData != null)
{
//get scores and intensities for right and left eye closing - 22 possible expressions --> put into hashtable
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
//this gets a list of enum names as strings
var enumNames = Enum.GetNames(typeof(PXCMFaceData.ExpressionsData.FaceExpression));
//for all enumnames, calculate the
for (int j = 0; j < enumNames.Length; j++)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult innerScore;
singleExprData.QueryExpression((PXCMFaceData.ExpressionsData.FaceExpression)(j), out innerScore);
//Console.WriteLine((PXCMFaceData.ExpressionsData.FaceExpression)(j));
exprTable[enumNames[j]] = innerScore.intensity;
}
//Attempt to write to file if there are any significant events
/* //check if everything is 0
bool significantEntry = false;
foreach (DictionaryEntry entry in exprTable)
{
if (Convert.ToInt32(entry.Value.ToString()) != 0)
{
significantEntry = true;
break;
}
}
if (significantEntry) */
writeSignificantToFile(exprTable, boundingRect, averageDepth, landmarkPoints, eulerAngles, quaternionAngles);
singleExprData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT, out score);
lEyeClosedIntensity = score.intensity;
singleExprData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT, out score);
rEyeClosedIntensity = score.intensity;
//eye closed logic -> will be reset in UI thread after some number of frames
if (lEyeClosedIntensity >= EYE_CLOSED_DETECT_THRESHOLD)
lEyeClosed = true;
if (rEyeClosedIntensity >= EYE_CLOSED_DETECT_THRESHOLD)
rEyeClosed = true;
}
// }
}
}
if (emotion != null)
{
int numFaces = emotion.QueryNumFaces();
for (int fid = 0; fid < numFaces; fid++)
{
//TODO - MULTIPLE FACE IMPLEMENTATION?
//retrieve all est data
PXCMEmotion.EmotionData[] arrData = new PXCMEmotion.EmotionData[10];
emotion.QueryAllEmotionData(fid, out arrData);
//find emotion with maximum evidence
int idx_outstanding_emotion = 0;
int max_evidence = arrData[0].evidence;
for (int k = 1; k < 7; k++)
{
if (arrData[k].evidence < max_evidence)
{
}
else
{
max_evidence = arrData[k].evidence;
idx_outstanding_emotion = k;
}
}
currentEmotion = arrData[idx_outstanding_emotion].eid;
//Console.WriteLine(currentEmotion.ToString());
emotionEvidence = max_evidence;
// Console.WriteLine(currentEmotion.ToString() + ":" + emotionEvidence.ToString());
}
}
// Update the user interface
UpdateUI(colorBitmap);
// Release the frame
if (handData != null) handData.Dispose();
// colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
senseManager.ReleaseFrame();
}
}
private void ProcessingHandThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
hand = senseManager.QueryHand();
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
// Get number of tracked hands
nhands = handData.QueryNumberOfHands();
if (nhands > 0)
{
// Retrieve hand identifier
handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId);
// Retrieve hand data
handData.QueryHandDataById(handId, out ihand);
// Retrieve all hand joint data
for (int i = 0; i < nhands; i++)
{
for (int j = 0; j < 0x20; j++)
{
PXCMHandData.JointData jointData;
ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
nodes[i][j] = jointData;
}
}
// Get world coordinates for tip of middle finger on the first hand in camera range
handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x;
handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y;
handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z;
// Retrieve gesture data
if (handData.IsGestureFired("spreadfingers", out gestureData))
{
gesture = Gesture.FingerSpread;
}
else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
gesture = Gesture.Pinch;
}
else if (handData.IsGestureFired("wave", out gestureData))
{
gesture = Gesture.Wave;
}
}
else
{
gesture = Gesture.Undefined;
}
// Get alert status
for (int i = 0; i < handData.QueryFiredAlertsNumber(); i++)
{
PXCMHandData.AlertData alertData;
if (handData.QueryFiredAlertData(i, out alertData) != pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; }
//Displaying last alert
switch (alertData.label)
{
case PXCMHandData.AlertType.ALERT_HAND_DETECTED:
detectionAlert = "Hand Detected";
detectionStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED:
detectionAlert = "Hand Not Detected";
detectionStatusOk = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_CALIBRATED:
calibrationAlert = "Hand Calibrated";
calibrationStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_NOT_CALIBRATED:
calibrationAlert = "Hand Not Calibrated";
calibrationStatusOk = false;
break;
case PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS:
bordersAlert = "Hand Inside Borders";
borderStatusOk = true;
break;
case PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS:
bordersAlert = "Hand Out Of Borders";
borderStatusOk = false;
break;
}
}
UpdateUI();
if (handData != null) handData.Dispose();
}
senseManager.ReleaseFrame();
}
}
private void ConfigureHandModule()
{
hand = senseManager.QueryHand();
handConfig = hand.CreateActiveConfiguration();
// handConfig.EnableGesture("spreadfingers");
// handConfig.EnableGesture("two_fingers_pinch_open");
// handConfig.EnableGesture("wave");
handConfig.EnableAllAlerts();
handConfig.ApplyChanges();
}
// Update is called once per frame
void Update()
{
/* Make sure SenseManager Instance is valid */
if (sm == null)
return;
/* Wait until any frame data is available */
if (sm.AcquireFrame (false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
return;
/* Retrieve hand tracking Module Instance */
handAnalyzer = sm.QueryHand ();
if (handAnalyzer != null) {
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput ();
if (_handData != null) {
_handData.Update ();
/* Retrieve Gesture Data to manipulate GUIText */
PXCMHandData.GestureData gestureData;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++)
if (_handData.QueryFiredGestureData (i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
DisplayGestures (gestureData);
/* Retrieve Alert Data to manipulate GUIText */
PXCMHandData.AlertData alertData;
for (int i=0; i<_handData.QueryFiredAlertsNumber(); i++)
if (_handData.QueryFiredAlertData (i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
ProcessAlerts (alertData);
/* Retrieve all joint Data */
for (int i = 0; i < _handData.QueryNumberOfHands (); i++) {
PXCMHandData.IHand _iHand;
if (_handData.QueryHandData (PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR) {
for (int j = 0; j < MaxJoints; j++) {
if (_iHand.QueryTrackedJoint ((PXCMHandData.JointType)j, out jointData [i] [j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
jointData [i] [j] = null;
}
if (!handList.ContainsKey (_iHand.QueryUniqueId ()))
handList.Add (_iHand.QueryUniqueId (), _iHand.QueryBodySide ());
}
}
/* Smoothen and Display the Data - Joints and Bones*/
DisplayJoints ();
}
_handData.Dispose ();
}
handAnalyzer.Dispose ();
sm.ReleaseFrame ();
RotateCam ();
}
// Use this for initialization
void Start()
{
handList = new Hashtable ();
/* Initialize a PXCMSenseManager instance */
sm = PXCMSenseManager.CreateInstance ();
if (sm == null)
Debug.LogError ("SenseManager Initialization Failed");
/* Enable hand tracking and retrieve an hand module instance to configure */
sts = sm.EnableHand ();
handAnalyzer = sm.QueryHand ();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError ("PXCSenseManager.EnableHand: " + sts);
/* Initialize the execution pipeline */
sts = sm.Init ();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError ("PXCSenseManager.Init: " + sts);
/* Retrieve the the DataSmoothing instance */
sm.QuerySession ().CreateImpl<PXCMDataSmoothing> (out ds);
/* Create a 3D Weighted algorithm */
smoother3D = new PXCMDataSmoothing.Smoother3D[MaxHands][];
/* Configure a hand - Enable Gestures and Alerts */
PXCMHandConfiguration hcfg = handAnalyzer.CreateActiveConfiguration ();
hcfg.EnableAllGestures ();
hcfg.EnableAlert (PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED);
hcfg.ApplyChanges ();
hcfg.Dispose ();
InitializeGameobjects ();
transform.RotateAround (transform.position, Vector3.up, 200 * Time.deltaTime);
// Vector3 _RotateAround = new Vector3 (1, 1f, 180f);
// transform.RotateAround (_RotateAround, Vector3.up, 20);
}
// Update is called once per frame
void Update () {
/* Make sure SenseManager Instance is valid */
if (senseManager == null)
return;
/* Wait until any frame data is available */
if (senseManager.AcquireFrame (false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
return;
/* Retrieve hand tracking Module Instance */
handAnalyzer = senseManager.QueryHand ();
try
{
if (handAnalyzer != null) {
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput ();
if (_handData != null) {
_handData.Update ();
PXCMHandData.IHand[] _iHand = new PXCMHandData.IHand[MaxHands];
NumOfHands = _handData.QueryNumberOfHands ();
hands.isLeft = false;
hands.isRight = false;
/* Retrieve all joint Data */
if (_handData.QueryHandData (PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out _iHand[0]) == pxcmStatus.PXCM_STATUS_NO_ERROR) {
/*Identify left/right hand */
gesture[0].isExist = true;
hands.isLeft = true;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++){
//Debug.Log (i.ToString());
if (_handData.QueryFiredGestureData (i, out gesture[0].gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.Log (gesture[0].gestureData.name);
//continue;//If you want to use this gesture info, you need to realize a SEND function here.
}
for (int j = 0; j < MaxJoints; j++) {
if (_iHand[0].QueryTrackedJoint ((PXCMHandData.JointType)j, out hands.jointData [0] [j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
hands.jointData [0] [j] = null;
/* hands.smoothPosition[0][j].AddSample(hands.jointData[0][j].positionWorld);
hands.smoothLocalRotation[0][j].AddSample(((Quaternion)hands.jointData[0][j].localRotation).eulerAngles);
hands.smoothGlobalRotation[0][j].AddSample(((Quaternion)hands.jointData[0][j].globalOrientation).eulerAngles);
*/}
if (!handList.ContainsKey (_iHand[0].QueryUniqueId ()))
handList.Add (_iHand[0].QueryUniqueId (), _iHand[0].QueryBodySide ());
}else{
gesture[0].isExist = false;
}
if (_handData.QueryHandData (PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out _iHand[1]) == pxcmStatus.PXCM_STATUS_NO_ERROR) {
/*Identify left/right hand */
gesture[1].isExist = true;
hands.isRight = true;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++)
if (_handData.QueryFiredGestureData (i, out gesture[1].gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
continue;//If you want to use this gesture info, you need to realize a SEND function here.
for (int j = 0; j < MaxJoints; j++) {
if (_iHand[1].QueryTrackedJoint ((PXCMHandData.JointType)j, out hands.jointData [1] [j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
hands.jointData [1] [j] = null;
}
if (!handList.ContainsKey (_iHand[1].QueryUniqueId ()))
handList.Add (_iHand[1].QueryUniqueId (), _iHand[1].QueryBodySide ());
}else{
gesture[1].isExist = false;
}
}
_handData.Dispose ();
}
}
catch (IOException ex)
{
Console.WriteLine("An IOException has been thrown!");
Console.WriteLine(ex.ToString());
Console.ReadLine();
return;
}
handAnalyzer.Dispose ();
senseManager.ReleaseFrame ();
}
private void Uninitialize()
{
if ( senseManager != null ) {
senseManager.Dispose();
senseManager = null;
}
if ( projection != null ) {
projection.Dispose();
projection = null;
}
if ( handData != null ) {
handData.Dispose();
handData = null;
}
if ( handAnalyzer != null ) {
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
// Update is called once per frame
void Update()
{
if (session == null)
{
return;
}
// For accessing hand data
handAnalyzer = session.QueryHand();
faceAnalyzer = session.QueryFace();
if (handAnalyzer != null)
{
PXCMHandData handData = handAnalyzer.CreateOutput();
if (handData != null)
{
handData.Update();
PXCMHandData.IHand IHAND; // Ihand instance for accessing future data
// Int32 IhandData; // for QueryOpenness Value
// PXCMPoint3DF32 location; // Stores hand tracking position
//Fills IHAND with information to later be grabbed and used for tracking + openness
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, 0, out IHAND);
// If there is data in Ihand
if (IHAND != null)
{
// Debug.DrawLine(transform.position, hit.point, Color.red);
// Inits hand tracking from the center of the hand.
// location = IHAND.QueryMassCenterWorld();
// if (mCurrentDart != null)
// {
// Vector3 locationUnity = new Vector3(location.x, location.y, location.z);
// mCurrentDart.transform.localPosition = locationUnity * RSScale;
// }
}
}
handAnalyzer.Dispose();
session.ReleaseFrame();
}
if (faceAnalyzer != null)
{
PXCMFaceData facedata = faceAnalyzer.CreateOutput();
if (facedata != null)
{
Int32 nfaces = facedata.QueryNumberOfDetectedFaces();
for (Int32 i = 0; i < nfaces; i++)
{
// Retrieve the face landmark data instance
PXCMFaceData.Face face = facedata.QueryFaceByIndex(i);
PXCMFaceData.PoseData pdata = face.QueryPose();
// retrieve the pose information
PXCMFaceData.PoseEulerAngles angles;
pdata.QueryPoseAngles(out angles);
Debug.Log("Eular Angles yaw : " + angles.yaw);
Debug.Log("Eular Angles pitch: " + angles.pitch);
Debug.Log("Eular Angles Roll: " + angles.roll);
angles.pitch = gameObject.transform.rotation.z;
angles.yaw = gameObject.transform.rotation.y;
}
// device is a PXCMCapture.Device instance
}
}
}
public void InitializeHandsStream()
{
manager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 640, 480, 30);
var s = manager.EnableHand();
this.handAnalyzer = manager.QueryHand();
var re = manager.Init();
this.outputData = handAnalyzer.CreateOutput();
if (re != pxcmStatus.PXCM_STATUS_NO_ERROR)
throw new RealSenseException("error: " + re, re);
this.isHandsStreamEnabled = true;
}
public MainWindow()
{
InitializeComponent();
//set the current date and time
currentDateTime = DateTime.Now.ToString("yyyyMMddHHmmssfff");
//set total timer count to 0 and init vars
highPerformanceTimer = new HiPerfTimer();
totalHighPerfTimeElapsed = 0;
numLinesWritten = 0; //set the total number of lines written to 0 so we can track when to start the timer
//init pipe stuff
pipeClient = new MyClient(PIPE_NAME);
pipeClient.SendMessage("I Am Intel RealSense");
//Debug.WriteLine("Server Ready");
//initialise combobox
populateComboBox();
//init the exprToDisplay global var
exprToDisplay = "";
//Work on the file
//create paths
string dirToCreate = "data";
string dirToCreateFull = System.IO.Path.GetFullPath(dirToCreate);
Directory.CreateDirectory(dirToCreateFull);
dirToCreate = "video";
dirToCreateFull = System.IO.Path.GetFullPath(dirToCreate);
Directory.CreateDirectory(dirToCreateFull);
//create the csv file to write to
file = new StreamWriter("data/" + currentDateTime + "data" + ".csv");
//initialise global expressions array - faster to add the keys here?
var enumListMain = Enum.GetNames(typeof(PXCMFaceData.ExpressionsData.FaceExpression));
exprTable = new Hashtable();
string initLine = "";
//Add the column schema
//Initial line: timestamp and high prec time
initLine += "TIMESTAMP,HIGH_PRECISION_TIME_FROM_START,STIMCODE";
//add all the expression data columns
for (int i = 0; i < enumListMain.Length; i++)
{
exprTable.Add(enumListMain[i], 0);
initLine += "," + enumListMain[i];
}
//add the bounding rectangle column
initLine += "," + "BOUNDING_RECTANGLE_HEIGHT" + "," + "BOUNDING_RECTANGLE_WIDTH" + "," + "BOUNDING_RECTANGLE_X" + "," + "BOUNDING_RECTANGLE_Y";
//add the average depth column
initLine += "," + "AVERAGE_DEPTH";
//add landmark points column
for (int i = 0; i < LANDMARK_POINTS_TOTAL; i++)
{
initLine += "," + "LANDMARK_" + i + "_X";
initLine += "," + "LANDMARK_" + i + "_Y";
}
//add euler angles columns
initLine += "," + "EULER_ANGLE_PITCH" + "," + "EULER_ANGLE_ROLL" + "," + "EULER_ANGLE_YAW";
initLine += "," + "QUATERNION_W" + "," + "QUATERNION_X" + "," + "QUATERNION_Y" + "," + "QUATERNION_Z";
//write the initial row to the file
file.WriteLine(initLine);
//configure the camera mode selection box
cbCameraMode.Items.Add("Color");
cbCameraMode.Items.Add("IR");
cbCameraMode.Items.Add("Depth");
//configure initial camera mode
cameraMode = "Color";
//initialise global vars
numFacesDetected = 0;
handWaving = false;
handTrigger = false;
handResetTimer = 0;
lEyeClosedIntensity = 0;
lEyeClosed = false;
lEyeClosedTrigger = false;
lEyeClosedResetTimer = 0;
rEyeClosed = false;
rEyeClosedTrigger = false;
rEyeClosedResetTimer = 0;
rEyeClosedIntensity = 0;
emotionEvidence = 0;
blinkTrigger = false;
blinkResetTimer = 0;
//global fps vars
prevTime = 0;
stopwatch = new Stopwatch();
// Instantiate and initialize the SenseManager
senseManager = PXCMSenseManager.CreateInstance();
if (senseManager == null)
{
MessageBox.Show("Cannot initialise sense manager: closing in 20s, report to Sriram");
Thread.Sleep(20000);
Environment.Exit(1);
}
//capture samples
senseManager.captureManager.SetFileName("video/" + currentDateTime + ".raw", true);
//Enable color stream
senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, STREAM_WIDTH, STREAM_HEIGHT, STREAM_FPS);
senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, STREAM_WIDTH, STREAM_HEIGHT, STREAM_FPS);
senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, STREAM_WIDTH, STREAM_HEIGHT, STREAM_FPS);
//Enable face and hand tracking AND EMOTION TRACKING
senseManager.EnableHand();
senseManager.EnableFace();
senseManager.EnableEmotion();
//Initialise the senseManager - begin collecting data
senseManager.Init();
// Configure the Hand Module
hand = senseManager.QueryHand();
handConfig = hand.CreateActiveConfiguration();
handConfig.EnableGesture("wave");
handConfig.EnableAllAlerts();
handConfig.ApplyChanges();
//Configure the Face Module
face = senseManager.QueryFace();
faceConfig = face.CreateActiveConfiguration();
faceConfig.EnableAllAlerts();
faceConfig.detection.isEnabled = true; //enables querydetection function to retrieve face loc data
faceConfig.detection.maxTrackedFaces = 1; //MAXIMUM TRACKING - 1 FACE
faceConfig.ApplyChanges();
//Configure the sub-face-module Expressions
exprConfig = faceConfig.QueryExpressions();
exprConfig.Enable();
exprConfig.EnableAllExpressions();
faceConfig.ApplyChanges();
// Start the worker thread that processes the captured data in real-time
processingThread = new Thread(new ThreadStart(ProcessingThread));
processingThread.Start();
}
// RS
private void STButton_Click(object sender, RoutedEventArgs e) {
EXButton.IsEnabled = true;
STButton.IsEnabled = false;
// init the sense manager
_senseManager = PXCMSenseManager.CreateInstance();
// enable hand analysis in the multimodal pipeline
_senseManager.EnableTouchlessController();
// init the pipeline
if (_senseManager.Init() < pxcmStatus.PXCM_STATUS_NO_ERROR) {
MessageBox.Show("RealSense Camera init failed");
}
// get an instance of the touchless control
_touchlessController = _senseManager.QueryTouchlessController();
// register fo ux event
_touchlessController.SubscribeEvent(OnTouchlessControllerUXEventHandler); //event
_touchlessController.SubscribeAlert(OnFiredAlertEventHandler); //alert
// configuration
pxcmStatus rc;
PXCMTouchlessController.ProfileInfo pInfo;
rc = _touchlessController.QueryProfile(out pInfo);
pInfo.config = PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Allow_Zoom
| PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Allow_Selection
| PXCMTouchlessController.ProfileInfo.Configuration.Configuration_Allow_Back;
rc = _touchlessController.SetProfile(pInfo);
// configure hand module
_hand = _senseManager.QueryHand();
_handConfig = _hand.CreateActiveConfiguration();
_handConfig.EnableGesture("thumb_up", true); // true is importment
_handConfig.EnableGesture("v_sign", true);
_handConfig.SubscribeGesture(OnFiredGestureEventHandler);
_handConfig.ApplyChanges();
// processing loop
_processingThread = new Thread(new ThreadStart(ProcessingThread));
_processingThread.Start();
}
// Update is called once per frame
void Update()
{
/* Make sure SenseManager Instance is valid */
if (sm == null)
{
return;
}
/* Wait until any frame data is available */
if (sm.AcquireFrame(false) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve hand tracking Module Instance */
handAnalyzer = sm.QueryHand();
if (handAnalyzer != null)
{
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput();
if (_handData != null)
{
_handData.Update();
/* Retrieve Gesture Data to manipulate GUIText */
PXCMHandData.GestureData gestureData;
for (int i = 0; i < _handData.QueryFiredGesturesNumber(); i++)
{
if (_handData.QueryFiredGestureData(i, out gestureData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
DisplayGestures(gestureData);
}
}
/* Retrieve Alert Data to manipulate GUIText */
PXCMHandData.AlertData alertData;
for (int i = 0; i < _handData.QueryFiredAlertsNumber(); i++)
{
if (_handData.QueryFiredAlertData(i, out alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
ProcessAlerts(alertData);
}
}
/* Retrieve all joint Data */
for (int i = 0; i < _handData.QueryNumberOfHands(); i++)
{
PXCMHandData.IHand _iHand;
if (_handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_FIXED, i, out _iHand) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
for (int j = 0; j < MaxJoints; j++)
{
if (_iHand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData[i][j]) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
jointData[i][j] = null;
}
}
if (!handList.ContainsKey(_iHand.QueryUniqueId()))
{
handList.Add(_iHand.QueryUniqueId(), _iHand.QueryBodySide());
}
}
}
/* Smoothen and Display the Data - Joints and Bones*/
DisplayJoints();
}
handAnalyzer.Dispose();
}
sm.ReleaseFrame();
RotateCam();
}
// 手の検出の初期化
private void InitializeHandTracking()
{
// 手の検出器を取得する
handAnalyzer = senseManager.QueryHand();
if ( handAnalyzer == null ) {
throw new Exception( "手の検出器の取得に失敗しました" );
}
// 手のデータを作成する
handData = handAnalyzer.CreateOutput();
if ( handData == null ) {
throw new Exception( "手の検出器の作成に失敗しました" );
}
// RealSense カメラであれば、プロパティを設定する
var device = senseManager.QueryCaptureManager().QueryDevice();
PXCMCapture.DeviceInfo dinfo;
device.QueryDeviceInfo( out dinfo );
if ( dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM ) {
device.SetDepthConfidenceThreshold( 1 );
//device.SetMirrorMode( PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED );
device.SetIVCAMFilterOption( 6 );
}
// 手の検出の設定
var config = handAnalyzer.CreateActiveConfiguration();
config.EnableSegmentationImage( true );
config.ApplyChanges();
config.Update();
}
public void SetManualMode()
{
// REALSENSE
/* Initialize a PXCMSenseManager instance */
sm = PXCMSenseManager.CreateInstance();
if (sm == null)
Debug.LogError("SenseManager Initialization Failed");
/* Enable hand tracking and retrieve an hand module instance to configure */
sts = sm.EnableHand();
handAnalyzer = sm.QueryHand();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError("PXCSenseManager.EnableHand: " + sts);
/* Initialize the execution pipeline */
sts = sm.Init();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
Debug.LogError("PXCSenseManager.Init: " + sts);
/* Retrieve the the DataSmoothing instance */
sm.QuerySession().CreateImpl<PXCMSmoother>(out smoother);
/* Create a 3D Weighted algorithm */
smoother3D = new PXCMSmoother.Smoother3D[MaxHands][];
/* Configure a hand - Enable Gestures and Alerts */
PXCMHandConfiguration hcfg = handAnalyzer.CreateActiveConfiguration();
if (hcfg != null)
{
hcfg.EnableAllGestures();
hcfg.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED);
hcfg.SubscribeGesture(OnFiredGesture);
hcfg.ApplyChanges();
hcfg.Dispose();
}
// handAnalyzer = sm.QueryHand();
//////
curMode = userMode = DroneMode.Manual;
targetBehavior = targetDirectBehavior;
EnterFreeMovementState();
}
pxcmStatus newHandFrame(PXCMHandModule hand)
{
if (hand != null)
{
PXCMHandData handData = hand.CreateOutput();
handData.Update();
PXCMHandData.IHand iHandDataLeft = null, iHandDataRight = null;
PXCMHandData.JointData jointData = null;
PXCMImage image = null;
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out iHandDataLeft);
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out iHandDataRight);
if (handForm != null && !handForm.IsDisposed)
{
this.handForm.HandCount = handData.QueryNumberOfHands();
if (iHandDataLeft != null)
{
iHandDataLeft.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.LeftHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
if (iHandDataRight != null)
{
iHandDataRight.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.RightHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
}
if (iHandDataLeft != null)
{
if (jointData == null)
{
iHandDataLeft.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (iHandDataRight != null)
{
if (jointData == null)
{
iHandDataRight.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (jointData != null && canTrack.Checked)
{
Cursor.Position = new System.Drawing.Point(
(int)((640.0f - jointData.positionImage.x) * Screen.PrimaryScreen.Bounds.Width / 640.0f),
(int)(jointData.positionImage.y * Screen.PrimaryScreen.Bounds.Height / 480.0f));
PXCMHandData.GestureData gestureData = null;
if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
Program.DoMouseClick();
}
Console.WriteLine("Z Position: " + jointData.positionWorld.z);
}
handData.Dispose();
}
return(pxcmStatus.PXCM_STATUS_NO_ERROR);
}
// Use this for initialization
void Start()
{
// Creates an instance of the sense manager to be called later
session = PXCMSenseManager.CreateInstance();
//Output an error if there is no instance of the sense manager
if (session == null)
{
Debug.LogError("SenseManager Init Failed!");
}
// Enables hand tracking
sts = session.EnableHand();
handAnalyzer = session.QueryHand();
sts2 = session.EnableFace();
faceAnalyzer = session.QueryFace();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCSenseManager.EnableHand: " + sts);
}
if (sts2 != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCSenseManager.EnableFace: " + sts2);
}
// Creates the session
sts = session.Init();
sts2 = session.Init();
if (sts != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCSenseManager.Init: " + sts);
}
if (sts2 != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Debug.LogError("PXCSenseManager.Init: " + sts2);
}
// Creates a hand config for future data
PXCMHandConfiguration handconfig = handAnalyzer.CreateActiveConfiguration();
PXCMFaceConfiguration faceconfig = faceAnalyzer.CreateActiveConfiguration();
//If there is handconfig instance
if (handconfig != null)
{
handconfig.EnableAllAlerts();
handconfig.ApplyChanges();
handconfig.Dispose();
}
if (faceconfig != null)
{
faceconfig.EnableAllAlerts();
faceconfig.ApplyChanges();
faceconfig.Dispose();
}
}
//Called every frame when in manual mode
private void UpdateFreeMovementState()
{
if (!manualEnabled)
return;
if (gesture.Equals("fist"))
{
GetComponent<Rigidbody>().AddForce(1 * transform.forward * forceMultiplier);
}
else if (gesture.Equals("v_sign"))
{
GetComponent<Rigidbody>().AddTorque(-1 * transform.up * forceMultiplier * rotateFactor);
gesture = "";
}
else if (gesture.Equals("thumb_down"))
{
GetComponent<Rigidbody>().AddTorque(1 * transform.up * forceMultiplier * rotateFactor);
gesture = "";
}
/* Retrieve hand tracking Module Instance */
handAnalyzer = sm.QueryHand();
/* Retrieve hand tracking Data */
PXCMHandData _handData = handAnalyzer.CreateOutput();
if (_handData != null)
{
_handData.Update();
}
handAnalyzer.Dispose();
sm.ReleaseFrame();
}
// 手の検出の初期化
private void InitializeHandTracking()
{
// 手の検出器を取得する
handAnalyzer = senseManager.QueryHand();
if ( handAnalyzer == null ) {
throw new Exception( "手の検出器の取得に失敗しました" );
}
// 手のデータを作成する
handData = handAnalyzer.CreateOutput();
if ( handData == null ) {
throw new Exception( "手の検出器の作成に失敗しました" );
}
// RealSense カメラであれば、プロパティを設定する
var device = senseManager.QueryCaptureManager().QueryDevice();
PXCMCapture.DeviceInfo dinfo;
device.QueryDeviceInfo( out dinfo );
if ( dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM ) {
device.SetDepthConfidenceThreshold( 1 );
//device.SetMirrorMode( PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED );
device.SetIVCAMFilterOption( 6 );
}
// 手の検出の設定
handConfig = handAnalyzer.CreateActiveConfiguration();
// 登録されているジェスチャーを列挙する
var num = handConfig.QueryGesturesTotalNumber();
for ( int i = 0; i < num; i++ ){
string gestureName;
var sts = handConfig.QueryGestureNameByIndex( i, out gestureName );
if ( sts == pxcmStatus.PXCM_STATUS_NO_ERROR ){
ComboGesture.Items.Add( gestureName );
}
}
handConfig.ApplyChanges();
handConfig.Update();
}
private void Uninitialize()
{
if ( senseManager != null ) {
senseManager.Dispose();
senseManager = null;
}
if ( handConfig != null ) {
handConfig.Dispose();
handConfig = null;
}
if ( handData != null ) {
handData.Dispose();
handData = null;
}
if ( handAnalyzer != null ) {
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
private void InitRealSense()
{
this.releaseInstances();
errorManager = new ErrorManager();
this.manager = PXCMSenseManager.CreateInstance();
if (manager == null)
{
errorManager.InitError("PXCMSenseManager::Create");
}
// Init hand module
errorManager.RunWithStatusCheck(() => manager.EnableHand(), "PXCMHandModule");
this.handModule = this.manager.QueryHand();
this.handData = this.handModule.CreateOutput();
// config hand module
var hconfig = this.handModule.CreateActiveConfiguration();
hconfig.EnableTrackedJoints(true);
//Gestures
hconfig.DisableAllGestures();
//alerts========================================================
hconfig.DisableAllAlerts();
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_CALIBRATED);
//hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_DETECTED);
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_NOT_DETECTED);
// For hand tracking
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_INSIDE_BORDERS);
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_OUT_OF_BORDERS);
// For user info
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_TOO_CLOSE);
hconfig.EnableAlert(PXCMHandData.AlertType.ALERT_HAND_TOO_FAR);
hconfig.SubscribeAlert(OnAlertFired);
//alerts=========================================================
hconfig.ApplyChanges();
hconfig.Dispose();
// Init touchless controller:: Move tracking
errorManager.RunWithStatusCheck(() => manager.EnableTouchlessController(), "TouchlessController");
this.touchController = this.manager.QueryTouchlessController();
// On processed frame callback
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler {
onModuleProcessedFrame = OnFrameProcessed
};
// UX events callback
this.touchController.SubscribeEvent(OnTouchlessControllerUxEvent);
// Init Sense manager
errorManager.RunWithStatusCheck(() => manager.Init(handler), "PXCMSenseManager::Init");
errorManager.RunWithStatusCheck(() => manager.StreamFrames(false), "PXCMSenseManager::StreamFrames");
if (this.manager.captureManager == null || this.manager.captureManager.device == null)
{
throw new NoDeviceException();
}
// Query scene info
PXCMCapture.Sample sample = null;
do {
sample = manager.QuerySample();
}
while (sample == null || sample.depth == null || sample.depth.info == null);
sceneXCenter = sample.depth.info.width / 2;
sceneYCenter = sample.depth.info.height / 2;
//Set delay before close gesture ability
Task.Factory.StartNew(() =>
{
Thread.Sleep(3000);
delayForCloseAchived = true;
this.handData.Update();
handInsideBorders = this.handData.QueryNumberOfHands() > 0;
});
// Hand tracking started.
started = true;
}
