private void Uninitialize()
{
if (senseManager != null)
{
senseManager.Dispose();
senseManager = null;
}
if (projection != null)
{
projection.Dispose();
projection = null;
}
if (handData != null)
{
handData.Dispose();
handData = null;
}
if (handAnalyzer != null)
{
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
/// <summary> 終了処理 </summary>
private void Uninitialize()
{
if (senseManager != null)
{
senseManager.Dispose();
senseManager = null;
}
if (projection != null)
{
projection.Dispose();
projection = null;
}
if (handData != null)
{
handData.Dispose();
handData = null;
}
if (handAnalyzer != null)
{
handAnalyzer.Dispose();
handAnalyzer = null;
}
if (config != null)
{
config.UnsubscribeGesture(OnFiredGesture);
config.Dispose();
}
}
//RealSenseメソッド-------------------------------------------------------------------
/// <summary> 機能の初期化 </summary>
private bool InitializeRealSense()
{
try
{
//SenseManagerを生成
senseManager = PXCMSenseManager.CreateInstance();
//カラーストリームの有効
var sts = senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, COLOR_WIDTH, COLOR_HEIGHT, COLOR_FPS);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Colorストリームの有効化に失敗しました");
}
// Depthストリームを有効にする
sts = senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH,
DEPTH_WIDTH, DEPTH_HEIGHT, DEPTH_FPS);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Depthストリームの有効化に失敗しました");
}
// 手の検出を有効にする
sts = senseManager.EnableHand();
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("手の検出の有効化に失敗しました");
}
//パイプラインを初期化する
//(インスタンスはInit()が正常終了した後作成されるので，機能に対する各種設定はInit()呼び出し後となる)
sts = senseManager.Init();
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("パイプラインの初期化に失敗しました");
}
//ミラー表示にする
senseManager.QueryCaptureManager().QueryDevice().SetMirrorMode(
PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL);
//デバイスを取得する
device = senseManager.captureManager.device;
//座標変換オブジェクトを作成
projection = device.CreateProjection();
// 手の検出の初期化
InitializeHandTracking();
return(true);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(false);
}
}
public static DS4CalibrationRecord Calibrate(PXCMCapture.Device device)
{
PXCMProjection projection = device.CreateProjection();
/* Get a calibration instance */
PXCMCalibration calib = projection.QueryInstance <PXCMCalibration>();
PXCMCalibration.StreamCalibration calibration;
PXCMCalibration.StreamTransform transformation;
calib.QueryStreamProjectionParameters(PXCMCapture.StreamType.STREAM_TYPE_COLOR, out calibration, out transformation);
float[] translation = transformation.translation;
DS4CalibrationRecord record = new DS4CalibrationRecord
{
DeviceCapture = new DS4CalibrationRecordInternal
{
ColorFocalLength = device.QueryColorFocalLength().toFloatArray(),
DepthFocalLength = device.QueryDepthFocalLength().toFloatArray(),
ColorFOV = device.QueryColorFieldOfView().toFloatArray(),
DepthFOV = device.QueryDepthFieldOfView().toFloatArray(),
ColorPrincipalPoint = device.QueryColorPrincipalPoint().toFloatArray(),
DepthPrincipalPoint = device.QueryDepthPrincipalPoint().toFloatArray(),
ColorWidth = Frame.COLOR_WIDTH,
ColorHeight = Frame.COLOR_HEIGHT,
DepthWidth = Frame.DEPTH_WIDTH,
DepthHeight = Frame.DEPTH_HEIGHT,
DepthStride = Frame.DEPTH_STRIDE,
LowConfValue = device.QueryDepthLowConfidenceValue(),
Extrinsics = new float[] { 1.0f,
0.0f,
0.0f,
translation[0],
0.0f,
1.0f,
0.0f,
translation[1],
0.0f,
0.0f,
1.0f,
translation[2],
0.0f,
0.0f,
0.0f,
1.0f }
},
API = "RSSDK"
};
return(record);
}
public void Dispose()
{
if (projection == null)
{
return;
}
projection.Dispose();
projection = null;
}
public Projection(PXCMSession session, PXCMCapture.Device device, PXCMImage.ImageInfo dinfo)
{
/* retrieve the invalid depth pixel values */
invalid_value = device.QueryDepthLowConfidenceValue();
/* Create the projection instance */
projection = device.CreateProjection();
uvmap = new PXCMPointF32[dinfo.width * dinfo.height];
}
private void Initialize()
{
try {
// SenseManagerを生成する
senseManager = PXCMSenseManager.CreateInstance();
// カラーストリームを有効にする
var sts = senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR,
COLOR_WIDTH, COLOR_HEIGHT, COLOR_FPS);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("カラーストリームの有効化に失敗しました");
}
// Depthストリームを有効にする
sts = senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH,
DEPTH_WIDTH, DEPTH_HEIGHT, DEPTH_FPS);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Depthストリームの有効化に失敗しました");
}
// 手の検出を有効にする
sts = senseManager.EnableHand();
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("手の検出の有効化に失敗しました");
}
// パイプラインを初期化する
sts = senseManager.Init();
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("初期化に失敗しました");
}
// デバイスを取得する
var device = senseManager.QueryCaptureManager().QueryDevice();
// ミラー表示にする
device.SetMirrorMode(
PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL);
// 座標変換オブジェクトを作成
projection = device.CreateProjection();
// 手の検出の初期化
InitializeHandTracking();
}
catch (Exception ex) {
MessageBox.Show(ex.Message);
Close();
}
}
public Projection(PXCMSession session, PXCMCapture.Device device, PXCMImage.ImageInfo dinfo, PXCMImage.ImageInfo cinfo)
{
/* retrieve the invalid depth pixel values */
invalid_value = device.QueryDepthLowConfidenceValue();
/* Create the projection instance */
projection = device.CreateProjection();
uvmap = new PXCMPointF32[dinfo.width * dinfo.height];
invuvmap = new PXCMPointF32[cinfo.width * cinfo.height];
}
void OnDisable()
{
//Disposses all modules
Initialized = false;
if (SenseManager == null)
{
return;
}
if (FaceModuleOutput != null)
{
FaceModuleOutput.Dispose();
FaceModuleOutput = null;
}
if (HandDataOutput != null)
{
SenseManager.PauseHand(true);
HandDataOutput.Dispose();
HandDataOutput = null;
}
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
ImageRgbOutput = null;
}
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
ImageDepthOutput = null;
}
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
ImageIROutput = null;
}
if (_projection != null)
{
_projection.Dispose();
_projection = null;
}
UvMap = null;
PointCloud = null;
SenseManager.Dispose();
SenseManager = null;
}
public Projection(PXCMSession session, PXCMCapture.Device device, PXCMImage.ImageInfo dinfo)
{
//: start ros serial node:
// rosPublisher.start("192.168.0.10");
/* Create the projection instance */
projection = device.CreateProjection();
height = dinfo.height;
width = dinfo.width;
numOfPixels = dinfo.width * dinfo.height;
UInt16 invalid_value = device.QueryDepthLowConfidenceValue();
obj_detector = new managed_obj_detector.ObjDetector(dinfo.width, dinfo.height, invalid_value);
coords = new PXCMPoint3DF32[numOfPixels];
rgb_ir_d_xyz_points = new managed_obj_detector.RgbIrDXyzPoint[numOfPixels];
}
public Projection(PXCMSession session, PXCMCapture.Device device, PXCMImage.ImageInfo dinfo)
{
//: start ros serial node:
// rosPublisher.start("192.168.0.10");
/* Create the projection instance */
projection = device.CreateProjection();
height = dinfo.height;
width = dinfo.width;
numOfPixels = dinfo.width * dinfo.height;
UInt16 invalid_value = device.QueryDepthLowConfidenceValue();
obj_detector = new managed_obj_detector.ObjDetector(dinfo.width, dinfo.height, invalid_value);
coords = new PXCMPoint3DF32[numOfPixels];
rgb_ir_d_xyz_points = new managed_obj_detector.RgbIrDXyzPoint[numOfPixels];
}
public void Dispose()
{
if (rosPublisher != null)
{
rosPublisher.Dispose();
rosPublisher = null;
}
if (projection != null)
{
projection.Dispose();
projection = null;
}
if (obj_detector != null)
{
obj_detector.Dispose();
obj_detector = null;
}
}
public void Dispose()
{
if (rosPublisher != null)
{
rosPublisher.Dispose();
rosPublisher = null;
}
if (projection != null)
{
projection.Dispose();
projection = null;
}
if (obj_detector != null)
{
obj_detector.Dispose();
obj_detector = null;
}
}
public double[,] GetDepthData()
{
Depth = null;
PXCMCapture.Sample Image = sm.QuerySample();
PXCMImage depth = Image.depth;
if (depth != null)
{
var DepthWidth = depth.info.width;
var DepthHeight = depth.info.height;
Device = sm.captureManager.QueryDevice();
Device.SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL);
Projection = Device.CreateProjection();
PXCMPoint3DF32[] Vertices = new PXCMPoint3DF32[DepthHeight * DepthWidth];
Status = Projection.QueryVertices(depth, Vertices);
Depth = new double[6, (DepthWidth * DepthHeight) / 100];
int j = 0;
for (int i = 0; i < DepthWidth * DepthHeight; i += 10)
{
Depth[0, j] = -Vertices[i].x / 10;
Depth[1, j] = Vertices[i].y / 10;
Depth[2, j] = Vertices[i].z / 10;
j++;
if (i % 640 == 0)
{
i += 6400;
}
}
Projection.Dispose();
}
sm.ReleaseFrame();
return(Depth);
}
protected override bool Initialize()
{
this.imageBuffer = new byte[this.width * this.height * BYTE_PER_PIXEL];
this.GetSessionAndSenseManager();
pxcmStatus sts;
if (FInMode[0] == Mode.Color)
{
this.EnableColorStream();
}
// Depthストリームを有効にする
this.EnableDepthStream();
// 手の検出を有効にする
sts = this.senseManager.EnableHand();
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("手の検出の有効化に失敗しました");
}
// パイプラインを初期化する
this.InitSenseManager();
// ミラー表示にする
this.GetDevice();
this.SetMirrorMode();
// 座標変換オブジェクトを作成
this.projection = this.device.CreateProjection();
// 手の検出の初期化
this.InitializeHandTracking(senseManager);
this.initialized = true;
return(true);
}
protected override void Uninitialize()
{
this.initialized = false;
if (this.projection != null)
{
this.projection.Dispose();
this.projection = null;
}
if (this.handData != null)
{
this.handData.Dispose();
this.handData = null;
}
if (this.handAnalyzer != null)
{
this.handAnalyzer.Dispose();
this.handAnalyzer = null;
}
base.Uninitialize();
}
void OnDisable()
{
//Disposses all modules
Initialized = false;
if (SenseManager == null)
{
return;
}
DisposeFunctions.ForEach(i => i.DynamicInvoke());
if (FaceModuleOutput != null)
{
FaceModuleOutput.Dispose();
FaceModuleOutput = null;
}
if (HandDataOutput != null)
{
SenseManager.PauseHand(true);
HandDataOutput.Dispose();
HandDataOutput = null;
}
if (BlobDataOutput != null)
{
SenseManager.PauseBlob(true);
BlobDataOutput.Dispose();
BlobDataOutput = null;
}
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
ImageRgbOutput = null;
}
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
ImageDepthOutput = null;
}
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
ImageIROutput = null;
}
if (Image3DSegmentationOutput != null)
{
Image3DSegmentationOutput.Dispose();
Image3DSegmentationOutput = null;
}
if (Projection != null)
{
Projection.Dispose();
Projection = null;
}
/* GZ
* if (BlobExtractor != null)
* {
* BlobExtractor.Dispose();
* BlobExtractor = null;
* } */
UvMap = null;
PointCloud = null;
SenseManager.Dispose();
SenseManager = null;
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
_sts = SenseManager.EnableBlob();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) ||
IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
SenseManager.Enable3DSeg();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
//Set different configurations:
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
if (faceModule == null)
{
throw new UnityException("QueryFace returned null");
}
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15))
{
Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1");
NumberOfDetectedFaces = 1;
}
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
if (expressionConfig == null)
{
throw new UnityException("QueryExpressions returned null");
}
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
if (handAnalysis == null)
{
throw new UnityException("QueryHand returned null");
}
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableStabilizer(true);
handConfiguration.DisableAllAlerts();
handConfiguration.EnableSegmentationImage(false);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
// Blob
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
PXCMBlobModule blobAnalysis = SenseManager.QueryBlob();
if (blobAnalysis == null)
{
throw new UnityException("QueryBlob returned null");
}
PXCMBlobConfiguration blobConfiguration = blobAnalysis.CreateActiveConfiguration();
if (blobConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
blobConfiguration.Update();
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.EnableStabilizer(true);
blobConfiguration.SetMaxDistance(50 * 10);
blobConfiguration.ApplyChanges();
blobConfiguration.Dispose();
BlobDataOutput = blobAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection & blob extractor
if (Projection == null)
{
Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
/* GZ
* if (BlobExtractor == null)
* {
* SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor);
* }*/
// Set initialization flag
Initialized = true;
}
private void Uninitialize()
{
if ( senseManager != null ) {
senseManager.Dispose();
senseManager = null;
}
if ( projection != null ) {
projection.Dispose();
projection = null;
}
if ( handData != null ) {
handData.Dispose();
handData = null;
}
if ( handAnalyzer != null ) {
handAnalyzer.Dispose();
handAnalyzer = null;
}
}
private void Initialize()
{
try {
// SenseManagerを生成する
senseManager = PXCMSenseManager.CreateInstance();
// カラーストリームを有効にする
var sts = senseManager.EnableStream( PXCMCapture.StreamType.STREAM_TYPE_COLOR,
COLOR_WIDTH, COLOR_HEIGHT, COLOR_FPS );
if ( sts < pxcmStatus.PXCM_STATUS_NO_ERROR ) {
throw new Exception( "カラーストリームの有効化に失敗しました" );
}
// Depthストリームを有効にする
sts = senseManager.EnableStream( PXCMCapture.StreamType.STREAM_TYPE_DEPTH,
DEPTH_WIDTH, DEPTH_HEIGHT, DEPTH_FPS );
if ( sts < pxcmStatus.PXCM_STATUS_NO_ERROR ) {
throw new Exception( "Depthストリームの有効化に失敗しました" );
}
// 手の検出を有効にする
sts = senseManager.EnableHand();
if ( sts < pxcmStatus.PXCM_STATUS_NO_ERROR ) {
throw new Exception( "手の検出の有効化に失敗しました" );
}
// パイプラインを初期化する
sts = senseManager.Init();
if ( sts < pxcmStatus.PXCM_STATUS_NO_ERROR ) {
throw new Exception( "初期化に失敗しました" );
}
// デバイスを取得する
var device = senseManager.QueryCaptureManager().QueryDevice();
// ミラー表示にする
device.SetMirrorMode(
PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL );
// 座標変換オブジェクトを作成
projection = device.CreateProjection();
// 手の検出の初期化
InitializeHandTracking();
}
catch ( Exception ex ) {
MessageBox.Show( ex.Message );
Close();
}
}
public void Dispose()
{
if (projection == null) return;
projection.Dispose();
projection = null;
}
/// <summary>
/// コンストラクタ
///
/// カメラ画像表示用のWritableBitmapを準備してImageに割り当て
/// カメラ制御用のオブジェクトを取得して
/// Colorカメラストリームを有効にしてカメラのフレーム取得を開始する
/// カメラキャプチャーするためのTaskを準備して開始
/// </summary>
public MainWindow()
{
// 画面コンポーネントを初期化
InitializeComponent();
// ジェスチャー画像表示用Imageを非表示にする
HiddenGestureImage();
// カメラ画像書き込み用のWriteableBitmapを準備してImageコントローラーにセット
m_ColorWBitmap = new WriteableBitmap(1920, 1080, 96.0, 96.0, PixelFormats.Bgr32, null);
ColorCameraImage.Source = m_ColorWBitmap;
m_HandSegmentWBitmap = new WriteableBitmap(640, 480, 96.0, 96.0, PixelFormats.Gray8, null);
HandSegmentImage.Source = m_HandSegmentWBitmap;
m_HandJointWBitmap = new WriteableBitmap(640, 480, 96.0, 96.0, PixelFormats.Bgra32, null);
HandJointtImage.Source = m_HandJointWBitmap;
// カメラ制御のオブジェクトを取得
m_Session = PXCMSession.CreateInstance();
m_Cm = m_Session.CreateSenseManager();
// Color,Depth,Irストリームを有効にする
m_Cm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 30);
m_Cm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 640, 480, 30);
m_Cm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 640, 480, 30);
// Hand Trackingを有効化と設定
m_Cm.EnableHand();
PXCMHandModule handModule = m_Cm.QueryHand();
PXCMHandConfiguration handConfig = handModule.CreateActiveConfiguration();
handConfig.SetTrackingMode(PXCMHandData.TrackingModeType.TRACKING_MODE_FULL_HAND); // FULL_HANDモード
handConfig.EnableSegmentationImage(true); // SegmentationImageの有効化
handConfig.EnableAllGestures(); // すべてのジェスチャーを補足
handConfig.SubscribeGesture(OnFiredGesture); // ジェスチャー発生時のコールバック関数をセット
handConfig.ApplyChanges();
// HandDataのインスタンスを作成
m_HandData = handModule.CreateOutput();
// カメラのフレーム取得開始
pxcmStatus initState = m_Cm.Init();
if (initState < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// エラー発生
MessageBox.Show(initState + "\nカメラ初期化に失敗しました。");
return;
}
// カメラ取得画像をミラーモードにする
m_Cm.captureManager.device.SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL);
// 座標変換のためのProjectionインスタンスを取得
m_Projection = m_Cm.QueryCaptureManager().QueryDevice().CreateProjection();
// カメラキャプチャーをするためのタスクを準備して起動
m_CameraCaptureTask = new Task(() => CaptureCameraProcess());
m_CameraCaptureTask.Start();
}
void OnDisable()
{
//Disposses all modules
Initialized = false;
if (SenseManager == null) return;
DisposeFunctions.ForEach( i=> i.DynamicInvoke());
if (FaceModuleOutput != null)
{
FaceModuleOutput.Dispose();
FaceModuleOutput = null;
}
if (HandDataOutput != null)
{
SenseManager.PauseHand(true);
HandDataOutput.Dispose();
HandDataOutput = null;
}
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
ImageRgbOutput = null;
}
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
ImageDepthOutput = null;
}
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
ImageIROutput = null;
}
if (Image3DSegmentationOutput != null)
{
Image3DSegmentationOutput.Dispose();
Image3DSegmentationOutput = null;
}
if (Projection != null)
{
Projection.Dispose();
Projection = null;
}
if (BlobExtractor != null)
{
BlobExtractor.Dispose();
BlobExtractor = null;
}
UvMap = null;
PointCloud = null;
SenseManager.Dispose();
SenseManager = null;
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) ||
IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
SenseManager.Enable3DSeg();
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true;
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (numberOfEnabledModalities > 0)
{
Debug.LogError("Unable to initialize all modalities");
}
return;
}
//Set different configurations:
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null) throw new UnityException("CreateActiveConfiguration returned null");
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15))
{
Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1");
NumberOfDetectedFaces = 1;
}
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null) throw new UnityException("CreateActiveConfiguration returned null");
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableStabilizer(true);
handConfiguration.DisableAllAlerts();
handConfiguration.EnableSegmentationImage(false);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find( i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection & blob extractor
if (Projection == null)
{
Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
if (BlobExtractor == null)
{
SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor);
}
// Set initialization flag
Initialized = true;
}
/// <summary>
/// Device-specific implementation of Connect.
/// Connects the camera.
/// </summary>
/// <remarks>This method is implicitely called by <see cref="Camera.Connect"/> inside a camera lock.</remarks>
/// <seealso cref="Camera.Connect"/>
protected override void ConnectImpl()
{
log.EnterMethod();
if (deviceInfo.Count == 0)
{
ScanForCameras();
}
if (deviceInfo.Count == 0)
{
log.Error(Name + ": No devices found.");
ExceptionBuilder.Throw(typeof(ConnectionFailedException), this, "No devices found.");
}
int deviceIdx;
for (deviceIdx = 0; deviceIdx < deviceInfo.Count; deviceIdx++)
{
if (deviceInfo[deviceIdx].name != "Intel(R) RealSense(TM) 3D Camera R200")
{
continue;
}
log.Debug("RealSense R200 found");
break;
}
if (deviceIdx >= deviceInfo.Count)
{
log.Error("Failed to find Intel Real Sense R200 camera!");
ExceptionBuilder.Throw(typeof(ConnectionFailedException), this, "No R200 found!");
}
ScanForProfiles(deviceIdx);
// Create an instance of the PXCSenseManager interface
pp = PXCMSenseManager.CreateInstance();
if (pp == null)
{
log.Error(Name + ": Failed to create an SDK pipeline object");
return;
}
pp.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, widthColor, heightColor, fpsColor);
pp.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, widthZImage, heightZImage, fpsZImage);
pxcmStatus retStat = pp.Init();
if (retStat < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
log.Error(Name + ": Init() failed.");
ExceptionBuilder.Throw(typeof(ConnectionFailedException), this, "Failed to Initialize Real Sense SDK");
}
pp.captureManager.device.ResetProperties(PXCMCapture.StreamType.STREAM_TYPE_ANY);
// Find calibration data
PXCMProjection projection = pp.captureManager.device.CreateProjection();
PXCMCalibration calib = projection.QueryInstance <PXCMCalibration>();
retStat = calib.QueryStreamProjectionParameters(PXCMCapture.StreamType.STREAM_TYPE_COLOR, out calibDataColor, out calibTransColor);
if (retStat < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
log.WarnFormat("Could not get calibration for color channel.");
}
retStat = calib.QueryStreamProjectionParameters(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, out calibDataDepth, out calibTransDepth);
if (retStat < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
log.WarnFormat("Could not get calibration for depth channel.");
}
projection.Dispose();
log.InfoFormat("Color calibration: f=({0}; {1}) c=({2}; {3}) k=({4}; {5}; {6}) p=({7}; {8})",
calibDataColor.focalLength.x, calibDataColor.focalLength.y,
calibDataColor.principalPoint.x, calibDataColor.principalPoint.y,
calibDataColor.radialDistortion[0], calibDataColor.radialDistortion[1], calibDataColor.radialDistortion[2],
calibDataColor.tangentialDistortion[0], calibDataColor.tangentialDistortion[1]
);
log.InfoFormat("Color translation: t=({0}; {1}; {2})",
calibTransColor.translation[0], calibTransColor.translation[1], calibTransColor.translation[2]
);
log.InfoFormat("Depth calibration: f=({0}; {1}) c=({2}; {3}) k=({4}; {5}; {6}) p=({7}; {8})",
calibDataDepth.focalLength.x, calibDataDepth.focalLength.y,
calibDataDepth.principalPoint.x, calibDataDepth.principalPoint.y,
calibDataDepth.radialDistortion[0], calibDataDepth.radialDistortion[1], calibDataDepth.radialDistortion[2],
calibDataDepth.tangentialDistortion[0], calibDataDepth.tangentialDistortion[1]
);
log.InfoFormat("Depth translation: t=({0}; {1}; {2})",
calibTransDepth.translation[0], calibTransDepth.translation[1], calibTransDepth.translation[2]
);
ActivateChannel(ChannelNames.ZImage);
ActivateChannel(ChannelNames.Color);
SelectChannel(ChannelNames.ZImage);
log.LeaveMethod();
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
//SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (numberOfEnabledModalities > 0)
{
print("Unable to initialize all modalities");
}
return;
}
//Set different configurations:
//SenseManager.QueryCaptureManager().device.SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED);
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableAllAlerts();
handConfiguration.EnableSegmentationImage(true);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
Start();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection
if (_projection == null)
{
_projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
// Set initialization flag
Initialized = true;
}
