private void ConfigureRealSense()
{
try
{
// Create the SenseManager instance
sm = PXCMSenseManager.CreateInstance();
// Enable the color stream
sm.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, ImageWidth, ImageHeight, 30);
// Enable person tracking
sm.EnablePersonTracking();
personModule = sm.QueryPersonTracking();
PXCMPersonTrackingConfiguration personConfig = personModule.QueryConfiguration();
personConfig.SetTrackedAngles(PXCMPersonTrackingConfiguration.TrackingAngles.TRACKING_ANGLES_ALL);
// Enable skeleton tracking - not supported on r200?
//PXCMPersonTrackingConfiguration.SkeletonJointsConfiguration skeletonConfig = personConfig.QuerySkeletonJoints();
//skeletonConfig.Enable();
// Enable the face module
sm.EnableFace();
PXCMFaceModule faceModule = sm.QueryFace();
PXCMFaceConfiguration faceConfig = faceModule.CreateActiveConfiguration();
faceConfig.SetTrackingMode(PXCMFaceConfiguration.TrackingModeType.FACE_MODE_COLOR_PLUS_DEPTH);
faceConfig.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
faceConfig.detection.maxTrackedFaces = 1;
faceConfig.ApplyChanges();
sm.EnableBlob();
PXCMBlobModule blobModule = sm.QueryBlob();
PXCMBlobConfiguration blobConfig = blobModule.CreateActiveConfiguration();
blobConfig.SetMaxBlobs(4); // 4 is the max
blobConfig.SetMaxDistance(2000); // in mm's
blobConfig.ApplyChanges();
//initialize the SenseManager
sm.Init();
faceData = faceModule.CreateOutput();
blobData = blobModule.CreateOutput();
// Mirror the image
sm.QueryCaptureManager().QueryDevice().SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL);
// Release resources
personConfig.Dispose();
faceConfig.Dispose();
faceModule.Dispose();
blobConfig.Dispose();
blobModule.Dispose();
}
catch (Exception)
{
MessageBox.Show("Unable to configure the RealSense camera. Please make sure a R200 camera is connected.", "System Error");
throw;
}
}
public MainWindow()
{
InitializeComponent();
// Configure RealSense session and SenseManager interface
session = PXCMSession.CreateInstance();
senseManager = session.CreateSenseManager();
senseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 30);
senseManager.EnableBlob();
senseManager.Init();
//Create blobModule from SenseManager
blobModule = senseManager.QueryBlob();
blobConfig = blobModule.CreateActiveConfiguration();
blobConfig.SetMaxBlobs(2);
blobConfig.SetMaxDistance(trackingDistance);
blobConfig.SetMaxObjectDepth(100);
blobConfig.SetMinPixelCount(400);
blobConfig.EnableColorMapping(true);
blobConfig.ApplyChanges();
blobData = blobModule.CreateOutput();
//Audio
WaveOut waveOut;
sineWaveProvider.SetWaveFormat(16000, 1);
sineWaveProvider.Frequency = 1000;
sineWaveProvider.Amplitude = 0;
waveOut = new WaveOut();
waveOut.Init(sineWaveProvider);
waveOut.Play();
startTime = DateTime.Now;
// Start Update thread
update = new Thread(new ThreadStart(Update));
update.Start();
}
void OnEnable()
{
Initialized = false;
/* Create a SenseManager instance */
SenseManager = PXCMSenseManager.CreateInstance();
if (SenseManager == null)
{
print("Unable to create the pipeline instance");
return;
}
if (_speechCommandsRef.Count != 0)
{
SetSenseOption(SenseOption.SenseOptionID.Speech);
}
int numberOfEnabledModalities = 0;
//Set mode according to RunMode - play from file / record / live stream
if (RunMode == MCTTypes.RunModes.PlayFromFile)
{
//CHECK IF FILE EXISTS
if (!System.IO.File.Exists(FilePath))
{
Debug.LogWarning("No Filepath Set Or File Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, false);
Debug.Log("SenseToolkitManager: Playing from file: " + FilePath);
}
}
if (RunMode == MCTTypes.RunModes.RecordToFile)
{
//CHECK IF PATH
string PathOnly = FilePath;
while (!PathOnly[PathOnly.Length - 1].Equals('\\'))
{
PathOnly = PathOnly.Remove(PathOnly.Length - 1, 1);
}
if (!System.IO.Directory.Exists(PathOnly))
{
Debug.LogWarning("No Filepath Set Or Path Doesn't Exist, Run Mode Will Be Changed to Live Stream");
RunMode = MCTTypes.RunModes.LiveStream;
}
else
{
PXCMCaptureManager cManager = SenseManager.QueryCaptureManager();
cManager.SetFileName(FilePath, true);
Debug.Log("SenseToolkitManager: Recording to file: " + FilePath);
}
}
/* Enable modalities according to the set options*/
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
SenseManager.EnableFace();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled = true;
SetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
_sts = SenseManager.EnableHand();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
_sts = SenseManager.EnableBlob();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
_sts = SenseManager.EnableTracker();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
if (!SpeechManager.IsInitialized)
{
if (SpeechManager.InitalizeSpeech())
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
else
{
UnsetSenseOption(SenseOption.SenseOptionID.Speech);
}
}
else
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled = true;
numberOfEnabledModalities++;
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoDepthStream, true) ||
IsSenseOptionSet(SenseOption.SenseOptionID.PointCloud, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_DEPTH, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoIRStream, true))
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_IR, 0, 0, 0);
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoColorStream, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled = true;
numberOfEnabledModalities++;
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.VideoSegmentation, true))
{
if (ColorImageQuality == MCTTypes.RGBQuality.FullHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1920, 1080, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 1280, 720, 0);
}
else if (ColorImageQuality == MCTTypes.RGBQuality.HalfHD)
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 960, 540, 0);
}
else
{
SenseManager.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 480, 0);
}
SenseManager.Enable3DSeg();
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Initialized = true;
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled = true;
numberOfEnabledModalities++;
}
/* Initialize the execution */
_sts = SenseManager.Init();
if (_sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
//Set different configurations:
// Face
if (IsSenseOptionSet(SenseOption.SenseOptionID.Face, true))
{
var faceModule = SenseManager.QueryFace();
if (faceModule == null)
{
throw new UnityException("QueryFace returned null");
}
var faceConfiguration = faceModule.CreateActiveConfiguration();
if (faceConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
faceConfiguration.Update();
faceConfiguration.detection.isEnabled = true;
faceConfiguration.detection.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.landmarks.isEnabled = true;
faceConfiguration.landmarks.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.pose.isEnabled = true;
faceConfiguration.pose.smoothingLevel = PXCMFaceConfiguration.SmoothingLevelType.SMOOTHING_DISABLED;
faceConfiguration.DisableAllAlerts();
faceConfiguration.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_APPEARANCE_TIME;
if ((NumberOfDetectedFaces < 1) || (NumberOfDetectedFaces > 15))
{
Debug.Log("Ilegal value for Number Of Detected Faces, value is set to 1");
NumberOfDetectedFaces = 1;
}
faceConfiguration.detection.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.landmarks.maxTrackedFaces = NumberOfDetectedFaces;
faceConfiguration.pose.maxTrackedFaces = NumberOfDetectedFaces;
PXCMFaceConfiguration.ExpressionsConfiguration expressionConfig = faceConfiguration.QueryExpressions();
if (expressionConfig == null)
{
throw new UnityException("QueryExpressions returned null");
}
expressionConfig.Enable();
expressionConfig.EnableAllExpressions();
faceConfiguration.ApplyChanges();
faceConfiguration.Dispose();
FaceModuleOutput = faceModule.CreateOutput();
UnsetSenseOption(SenseOption.SenseOptionID.VideoColorStream);
}
// Hand
if (IsSenseOptionSet(SenseOption.SenseOptionID.Hand, true))
{
PXCMHandModule handAnalysis = SenseManager.QueryHand();
if (handAnalysis == null)
{
throw new UnityException("QueryHand returned null");
}
PXCMHandConfiguration handConfiguration = handAnalysis.CreateActiveConfiguration();
if (handConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
handConfiguration.Update();
handConfiguration.EnableAllGestures();
handConfiguration.EnableStabilizer(true);
handConfiguration.DisableAllAlerts();
handConfiguration.EnableSegmentationImage(false);
handConfiguration.ApplyChanges();
handConfiguration.Dispose();
HandDataOutput = handAnalysis.CreateOutput();
}
// Blob
if (IsSenseOptionSet(SenseOption.SenseOptionID.Blob, true))
{
PXCMBlobModule blobAnalysis = SenseManager.QueryBlob();
if (blobAnalysis == null)
{
throw new UnityException("QueryBlob returned null");
}
PXCMBlobConfiguration blobConfiguration = blobAnalysis.CreateActiveConfiguration();
if (blobConfiguration == null)
{
throw new UnityException("CreateActiveConfiguration returned null");
}
blobConfiguration.Update();
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.EnableStabilizer(true);
blobConfiguration.SetMaxDistance(50 * 10);
blobConfiguration.ApplyChanges();
blobConfiguration.Dispose();
BlobDataOutput = blobAnalysis.CreateOutput();
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Object, true))
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled != true)
{
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Object).Enabled = true;
OnDisable();
OnEnable();
}
}
if (IsSenseOptionSet(SenseOption.SenseOptionID.Speech, true))
{
UpdateSpeechCommands();
SpeechManager.Start();
}
// Create an instance for the projection & blob extractor
if (Projection == null)
{
Projection = SenseManager.QueryCaptureManager().QueryDevice().CreateProjection();
}
/* GZ
* if (BlobExtractor == null)
* {
* SenseManager.session.CreateImpl<PXCMBlobExtractor>(out BlobExtractor);
* }*/
// Set initialization flag
Initialized = true;
}
/// <summary>
/// Using PXCMSenseManager to handle data
/// </summary>
public void SimplePipeline()
{
session = PXCMSession.CreateInstance();
if (session != null)
{
instance = session.CreateSenseManager();
if (instance == null)
{
Debug.Log("Create SenseManager Failure");
return;
}
pxcmStatus status = instance.EnableBlob();
PXCMBlobModule blobModule = instance.QueryBlob();
if (status != pxcmStatus.PXCM_STATUS_NO_ERROR || blobModule == null)
{
Debug.Log("Failed Loading Module");
return;
}
blobConfiguration = blobModule.CreateActiveConfiguration();
blobData = blobModule.CreateOutput();
if (blobConfiguration != null)
{
blobConfiguration.SetSegmentationSmoothing(1.0f);
blobConfiguration.SetMaxDistance(550);
blobConfiguration.SetMaxObjectDepth(100);
blobConfiguration.SetMaxBlobs(_maxBlobToShow);
blobConfiguration.SetContourSmoothing(1.0f);
blobConfiguration.EnableContourExtraction(true);
blobConfiguration.EnableSegmentationImage(true);
blobConfiguration.ApplyChanges();
}
if (blobData == null)
{
Debug.Log("Failed Create Output");
return;
}
PXCMSenseManager.Handler handler = new PXCMSenseManager.Handler();
handler.onModuleProcessedFrame = new PXCMSenseManager.Handler.OnModuleProcessedFrameDelegate(OnNewFrame);
if (instance.Init(handler) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.DeviceInfo dinfo;
instance.QueryCaptureManager().QueryDevice().QueryDeviceInfo(out dinfo);
if (dinfo != null && dinfo.model == PXCMCapture.DeviceModel.DEVICE_MODEL_IVCAM)
{
instance.QueryCaptureManager().QueryDevice().SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED);
}
/* Set the depth stream confidence threshold value - Any depth pixels with a confidence score below the threshold will be set to the low confidence pixel value*/
instance.QueryCaptureManager().QueryDevice().SetDepthConfidenceThreshold(10);
/* Set the smoothing aggressiveness parameter - High smoothing effect for distances between 850mm to 1000mm bringing good accuracy with moderate sharpness level.*/
instance.QueryCaptureManager().QueryDevice().SetIVCAMFilterOption(6);
}
}
else
{
Debug.Log("Init Failed");
}
}