public void StreamColorDepth() /* Stream Color and Depth Synchronously or Asynchronously */
{
try
{
bool sts = true;
/* Create an instance of the PXCMSenseManager interface */
PXCMSenseManager sm = PXCMSenseManager.CreateInstance();
if (sm == null)
{
SetStatus("Failed to create an SDK pipeline object");
return;
}
/* Optional: if playback or recoridng */
if ((Playback || Record) && File != null)
{
sm.captureManager.SetFileName(File, Record);
}
/* Optional: Set Input Source */
if (!Playback && DeviceInfo != null)
{
sm.captureManager.FilterByDeviceInfo(DeviceInfo);
}
/* Set Color & Depth Resolution and enable streams */
if (StreamProfileSet != null)
{
/* Optional: Filter the data based on the request */
sm.captureManager.FilterByStreamProfiles(StreamProfileSet);
/* Enable raw data streaming for specific stream types */
for (int s = 0; s < PXCMCapture.STREAM_LIMIT; s++)
{
PXCMCapture.StreamType st = PXCMCapture.StreamTypeFromIndex(s);
PXCMCapture.Device.StreamProfile info = StreamProfileSet[st];
if (info.imageInfo.format != 0)
{
/* For simple request, you can also use sm.EnableStream(...) */
PXCMVideoModule.DataDesc desc = new PXCMVideoModule.DataDesc();
desc.streams[st].frameRate.min = desc.streams[st].frameRate.max = info.frameRate.max;
desc.streams[st].sizeMin.height = desc.streams[st].sizeMax.height = info.imageInfo.height;
desc.streams[st].sizeMin.width = desc.streams[st].sizeMax.width = info.imageInfo.width;
desc.streams[st].options = info.options;
sm.EnableStreams(desc);
}
}
}
/* Initialization */
Timer timer = new Timer();
timer.UpdateStatus += UpdateStatus;
SetStatus("Init Started");
if (sm.Init() >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
/* Reset all properties */
sm.captureManager.device.ResetProperties(PXCMCapture.StreamType.STREAM_TYPE_ANY);
/* Set mirror mode */
PXCMCapture.Device.MirrorMode mirror = Mirror ? PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL : PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
sm.captureManager.device.SetMirrorMode(mirror);
SetStatus("Streaming");
while (!Stop)
{
/* Wait until a frame is ready: Synchronized or Asynchronous */
if (sm.AcquireFrame(Synced).IsError())
{
break;
}
/* Display images */
PXCMCapture.Sample sample = sm.QuerySample();
/* Render streams */
EventHandler <RenderFrameEventArgs> render = RenderFrame;
PXCMImage image = null;
if (ColorPanel != PXCMCapture.StreamType.STREAM_TYPE_ANY && render != null)
{
image = sample[ColorPanel];
render(this, new RenderFrameEventArgs(0, image));
}
if (DepthPanel != PXCMCapture.StreamType.STREAM_TYPE_ANY && render != null)
{
render(this, new RenderFrameEventArgs(1, sample[DepthPanel]));
}
if (IRPanel != PXCMCapture.StreamType.STREAM_TYPE_ANY && render != null)
{
render(this, new RenderFrameEventArgs(2, sample[IRPanel]));
}
if (IRLeftPanel != PXCMCapture.StreamType.STREAM_TYPE_ANY && render != null)
{
render(this, new RenderFrameEventArgs(3, sample[IRLeftPanel]));
}
if (IRRightPanel != PXCMCapture.StreamType.STREAM_TYPE_ANY && render != null)
{
render(this, new RenderFrameEventArgs(4, sample[IRRightPanel]));
}
/* Optional: Set Mirror State */
mirror = Mirror ? PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL : PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
if (mirror != sm.captureManager.device.QueryMirrorMode())
{
sm.captureManager.device.SetMirrorMode(mirror);
}
/* Optional: Show performance tick */
if (image != null)
{
timer.Tick(PXCMImage.PixelFormatToString(image.info.format) + " " + image.info.width + "x" + image.info.height);
}
sm.ReleaseFrame();
}
}
else
{
SetStatus("Init Failed");
sts = false;
}
sm.Dispose();
if (sts)
{
SetStatus("Stopped");
}
}
catch (Exception e) {
SetStatus(e.GetType().ToString());
}
}
private bool UpdateFace(AnimationTrigger animationTrigger)
{
animationTrigger.IsAnimationDataValid = false;
// Get the closest face
PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);
if (pxcmFace == null)
{
return(false);
}
if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
{
// PXCMFaceData.QueryFaceByIndex(0) failed.
return(false);
}
PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
if (pxcmLandmarksData == null)
{
// PXCMFaceData.Face.QueryLandmarks() failed.
return(false);
}
Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();
PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[]
{
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
};
foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
{
PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);
if (landmarkPoint == null)
{
// PXCMFaceData.LandmarksData.QueryPointIndex() failed.
return(false);
}
landmarkPoints.Add(landmarkType, landmarkPoint);
}
PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
if (pxcmExpressionsData == null)
{
// PXCMFaceData.Face.QueryExpressions() failed.
return(false);
}
animationTrigger.IsAnimationDataValid = true;
PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();
if (pxcmCaptureManager != null)
{
PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
if (pxcmCaptureDevice != null)
{
mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
}
}
animationTrigger.Animations.Clear();
float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;
float leftEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float leftEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
}
else
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
}
// Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
// Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));
PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();
if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
}
else
{
// Error querying expression: EXPRESSION_SMILE.
return(false);
}
return(true);
}
