private void TrackUserExpressions(PXCMFaceData.Face faceDataFace)
{
if (faceDataFace == null)
{
ResetUserTrackData();
return;
}
PXCMFaceData.ExpressionsData expressionsData = faceDataFace.QueryExpressions();
PXCMFaceData.ExpressionsData.FaceExpressionResult faceExpressionKiss;
PXCMFaceData.ExpressionsData.FaceExpressionResult faceExpressionTongueOut;
PXCMFaceData.ExpressionsData.FaceExpressionResult faceExpressionSmile;
PXCMFaceData.ExpressionsData.FaceExpressionResult faceExpressionMouthOpen;
if (expressionsData != null &&
expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_KISS, out faceExpressionKiss) &&
expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_TONGUE_OUT, out faceExpressionTongueOut) &&
expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out faceExpressionSmile) &&
expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN, out faceExpressionMouthOpen))
{
IsUserTracked = true;
UserExpressionKiss = faceExpressionKiss.intensity == 100;
UserExpressionTongueOut = faceExpressionTongueOut.intensity == 100;
UserExpressionSmile = faceExpressionSmile.intensity > 55;
UserExpressionMouthOpen = faceExpressionMouthOpen.intensity > 55;
}
else
{
ResetUserTrackData();
}
}
public void DrawExpressions(PXCMFaceData.Face face)
{
Debug.Assert(face != null);
if (m_bitmap == null || !Expressions.Checked)
{
return;
}
PXCMFaceData.ExpressionsData expressionsOutput = face.QueryExpressions();
if (expressionsOutput == null)
{
return;
}
lock (m_bitmapLock)
{
using (Graphics graphics = Graphics.FromImage(m_bitmap))
using (var brush = new SolidBrush(m_faceTextOrganizer.Colour))
{
const int imageSizeWidth = 18;
const int imageSizeHeight = 18;
int positionX = m_faceTextOrganizer.ExpressionsLocation.X;
int positionXText = positionX + imageSizeWidth;
int positionY = m_faceTextOrganizer.ExpressionsLocation.Y;
int positionYText = positionY + imageSizeHeight / 4;
foreach (var expressionEntry in m_expressionDictionary)
{
PXCMFaceData.ExpressionsData.FaceExpression expression = expressionEntry.Key;
PXCMFaceData.ExpressionsData.FaceExpressionResult result;
bool status = expressionsOutput.QueryExpression(expression, out result);
if (!status)
{
continue;
}
Bitmap cachedExpressionBitmap;
bool hasCachedExpressionBitmap = m_cachedExpressions.TryGetValue(expression, out cachedExpressionBitmap);
if (!hasCachedExpressionBitmap)
{
cachedExpressionBitmap = (Bitmap)m_resources.GetObject(expressionEntry.Value);
m_cachedExpressions.Add(expression, cachedExpressionBitmap);
}
using (var font = new Font(FontFamily.GenericMonospace, m_faceTextOrganizer.FontSize, FontStyle.Bold))
{
Debug.Assert(cachedExpressionBitmap != null, "cachedExpressionBitmap != null");
graphics.DrawImage(cachedExpressionBitmap, new Rectangle(positionX, positionY, imageSizeWidth, imageSizeHeight));
var expressionText = String.Format("= {0}", result.intensity);
graphics.DrawString(expressionText, font, brush, positionXText, positionYText);
positionY += imageSizeHeight;
positionYText += imageSizeHeight;
}
}
}
}
}
private Direction GetEyesDirection(PXCMFaceData.ExpressionsData data)
{
if ((DateTime.Now - _lastEyesDirectionDetection).TotalMilliseconds < 500)
{
return(_camera.Face.EyesDirection);
}
_lastEyesDirectionDetection = DateTime.Now;
var up = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_UP);
var down = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_DOWN);
var left = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_TURN_LEFT);
var right = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_TURN_RIGHT);
if (up > _eyesThresholds[Direction.Up])
{
_eyesThresholds[Direction.Up] = up * 0.7;
return(Direction.Up);
}
if (down > _eyesThresholds[Direction.Down])
{
_eyesThresholds[Direction.Down] = down * 0.7;
return(Direction.Down);
}
if (left > _eyesThresholds[Direction.Left])
{
_eyesThresholds[Direction.Left] = left * 0.7;
return(Direction.Left);
}
if (right > _eyesThresholds[Direction.Right])
{
_eyesThresholds[Direction.Right] = right * 0.7;
return(Direction.Right);
}
return(Direction.None);
}
private void SaveFacialExpressionData(PXCMFaceData.Face face)
{
FacialExpression fe = new FacialExpression();
PXCMFaceData.ExpressionsData edata = face.QueryExpressions();
if (edata == null)
{
#if DEBUG
Console.WriteLine("no expression this frame");
#endif
return;
}
#if DEBUG
else
{
Console.WriteLine("catch expression");
}
#endif
for (int i = 0; i < 22; i++)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
edata.QueryExpression((PXCMFaceData.ExpressionsData.FaceExpression)i, out score);
fe.facialExpressionIndensity[i] = score.intensity;
}
//dbhelper.saveEntity(fe);
}
private int GetFaceExpressionIntensity(PXCMFaceData.ExpressionsData data, FaceExpression faceExpression)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
data.QueryExpression(faceExpression, out score);
//if (score.intensity > 0) Debug.WriteLine(faceExpression + ":" +score.intensity);
return(score.intensity);
}
public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Face))
{
trigger.ErrorDetected = true;
return(false);
}
if (!(trigger is EventTrigger))
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
trigger.Source = this.name;
if (SenseToolkitManager.Instance.Initialized
&&
SenseToolkitManager.Instance.FaceModuleOutput != null)
{
int currentNumberOfFaces = SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces();
if (currentNumberOfFaces > FaceIndex)
{
PXCMFaceData.Face face = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(FaceIndex);
PXCMFaceData.ExpressionsData expressionData = face.QueryExpressions();
PXCMFaceData.ExpressionsData.FaceExpressionResult faceExpressionResult;
expressionData.QueryExpression(FacialExpression, out faceExpressionResult);
if (faceExpressionResult.intensity >= FireOverIntensity)
{
_lastFrameDetected = true;
_lastTimeDetected = Time.timeSinceLevelLoad;
return(false);
}
}
}
else
{
success = false;
}
if (Time.timeSinceLevelLoad - _lastTimeDetected > LostThreshold)
{
if (_lastFrameDetected)
{
_lastFrameDetected = false;
success = true;
}
}
return(success);
}
private void TrackExpressions(PXCMFaceData.Face face)
{
PXCMFaceData.ExpressionsData data = face.QueryExpressions();
if (data == null)
{
return;
}
Face.Mouth.IsSmiling = CheckFaceExpression(data, FaceExpression.EXPRESSION_SMILE, 40);
Face.Mouth.IsOpen = CheckFaceExpression(data, FaceExpression.EXPRESSION_MOUTH_OPEN, 15);
Face.LeftEye.IsOpen = !CheckFaceExpression(data, FaceExpression.EXPRESSION_EYES_CLOSED_LEFT, 15);
Face.RightEye.IsOpen = !CheckFaceExpression(data, FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT, 15);
Face.EyesDirection = GetEyesDirection(data);
}
public void Loop(LoopObjects loopObjects)
{
PXCMFaceData.Face face = loopObjects.Get <PXCMFaceData.Face>();
PXCMFaceData.ExpressionsData data = face?.QueryExpressions();
if (data == null)
{
return;
}
_camera.Face.Mouth.IsSmiling = CheckFaceExpression(data, FaceExpression.EXPRESSION_SMILE, SmileThreshold);
_camera.Face.Mouth.IsOpen = CheckFaceExpression(data, FaceExpression.EXPRESSION_MOUTH_OPEN, MonthOpenThreshold);
_camera.Face.LeftEye.IsOpen = !CheckFaceExpression(data, FaceExpression.EXPRESSION_EYES_CLOSED_LEFT, EyesClosedThreshold);
_camera.Face.RightEye.IsOpen = !CheckFaceExpression(data, FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT, EyesClosedThreshold);
_camera.Face.EyesDirection = GetEyesDirection(data);
}
private void TrackExpressions(PXCMFaceData.Face faceDataFace)
{
PXCMFaceData.ExpressionsData expressionsData = faceDataFace.QueryExpressions();
PXCMFaceData.ExpressionsData.FaceExpressionResult faceExpressionResult;
if (expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_KISS, out faceExpressionResult))
{
UserExpressions.Tracked = true;
UserExpressions.Kiss = faceExpressionResult.intensity == 100;
}
else
{
UserExpressions.Reset();
}
if (expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_TONGUE_OUT, out faceExpressionResult))
{
UserExpressions.Tracked = true;
UserExpressions.TongueOut = faceExpressionResult.intensity == 100;
}
else
{
UserExpressions.Reset();
}
if (expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out faceExpressionResult))
{
UserExpressions.Tracked = true;
UserExpressions.Smile = faceExpressionResult.intensity > 55;
Console.WriteLine("UserExpressions.Smile.intensity:" + faceExpressionResult.intensity.ToString());
}
else
{
UserExpressions.Reset();
}
if (expressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN, out faceExpressionResult))
{
UserExpressions.Tracked = true;
UserExpressions.MouthOpen = faceExpressionResult.intensity > 55;
Console.WriteLine("UserExpressions.MouthOpen.intensity:" + faceExpressionResult.intensity.ToString());
}
else
{
UserExpressions.Reset();
}
}
public override void Work(Graphics g)
{
// Retrieve gesture data
hand = senseManager.QueryHand();
face = senseManager.QueryFace(); //
if (hand != null)
{
// Retrieve the most recent processed data
handData = hand.CreateOutput();
handData.Update();
handWaving = handData.IsGestureFired("wave", out gestureData);
}
if (face != null)
{
faceData = face.CreateOutput();
faceData.Update();
//surching faces
Int32 nfaces = faceData.QueryNumberOfDetectedFaces();
for (Int32 i = 0; i < nfaces; i++)
{
// Retrieve the data instance
PXCMFaceData.Face faceI = faceData.QueryFaceByIndex(i);
PXCMFaceData.ExpressionsData edata = faceI.QueryExpressions();
if (edata != null)
{
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out score);
if (score.intensity >= 25)
{
smiling = true;
}
else
{
smiling = false;
}
Console.WriteLine(i + ": " + score.intensity);
}
}
faceData.Dispose();
}
}
/// <summary>
/// 获取实时面部表情
/// </summary>
/// <returns></returns>
public FacialExpression GetExpression()
{
int nFace = faceData.QueryNumberOfDetectedFaces();
if (nFace == 0)
{
#if DEBUG
//Console.WriteLine("No face in current frame");
#endif
return(null);
}
this.face = this.faceData.QueryFaceByIndex(0);
if (face == null)
{
return(null);
}
FacialExpression fe = new FacialExpression();
PXCMFaceData.ExpressionsData edata = face.QueryExpressions();
if (edata == null)
{
#if DEBUG
//Console.WriteLine("no expression this frame");
#endif
return(null);
}
#if DEBUG
else
{
//Console.WriteLine("catch expression");
}
#endif
for (int i = 0; i < 22; i++)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
edata.QueryExpression((PXCMFaceData.ExpressionsData.FaceExpression)i, out score);
fe.facialExpressionIndensity[i] = score.intensity;
}
return(fe);
}
private Direction GetEyesDirection(PXCMFaceData.ExpressionsData data)
{
if ((DateTime.Now - _lastEyesDirectionDetection).TotalMilliseconds < 500)
{
return(Face.EyesDirection);
}
_lastEyesDirectionDetection = DateTime.Now;
var up = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_UP);
var down = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_DOWN);
var left = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_TURN_LEFT);
var right = GetFaceExpressionIntensity(data, FaceExpression.EXPRESSION_EYES_TURN_RIGHT);
//Debug.WriteLine("U:{0}/{4} D:{1}/{5} L:{2}/{6} R:{3}/{7}", up, down, left, right,
// _eyesThresholds[Direction.Up],
// _eyesThresholds[Direction.Down],
// _eyesThresholds[Direction.Left],
// _eyesThresholds[Direction.Right]
// );
if (up > _eyesThresholds[Direction.Up])
{
_eyesThresholds[Direction.Up] = up * 0.7;
return(Direction.Up);
}
if (down > _eyesThresholds[Direction.Down])
{
_eyesThresholds[Direction.Down] = down * 0.7;
return(Direction.Down);
}
if (left > _eyesThresholds[Direction.Left])
{
_eyesThresholds[Direction.Left] = left * 0.7;
return(Direction.Left);
}
if (right > _eyesThresholds[Direction.Right])
{
_eyesThresholds[Direction.Right] = right * 0.7;
return(Direction.Right);
}
return(Direction.None);
}
private void SaveFeature(PXCMFaceData.Face qface, int frameCount)
{
PXCMFaceData.LandmarksData ldata = qface.QueryLandmarks();
PXCMFaceData.ExpressionsData edata = qface.QueryExpressions();
PXCMRectI32 rect;
// get the landmark data
var landmarkGroupTypes = Enum.GetValues(typeof(PXCMFaceData.LandmarksGroupType)).Cast <PXCMFaceData.LandmarksGroupType>();
qface.QueryDetection().QueryBoundingRect(out rect);
//保存时间戳
string time = DateTime.Now.ToString("yyyy-MM-dd") + " " + DateTime.Now.ToString("HH:mm:ss:fff");
sw_feature.Write(frameCount.ToString() + ' ' + time + ' ');
// 对于每个LandmarkPoint转换成成员变量中的world属性
PXCMFaceData.LandmarkPoint[] points;
ldata.QueryPoints(out points);
for (int i = 0; i < 78; i++)
{
//把脸部特征放入数组中
double position_x = Math.Round(points[i].image.x - rect.x, 4);
double position_y = Math.Round(points[i].image.y - rect.y, 4);
double position_z = Math.Round(points[i].world.z, 5);
sw_feature.Write(position_x.ToString() + ' ' + position_y.ToString() + ' ' + position_z.ToString() + ' ');
}
for (int i = 0; i < 22; i++)
{
double emotion = 0;
if (edata != null)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
edata.QueryExpression((PXCMFaceData.ExpressionsData.FaceExpression)i, out score);
Expression.facialExpressionIndensity[i] = score.intensity;
emotion = score.intensity;
}
sw_feature.Write(emotion.ToString() + ' ');
}
sw_feature.WriteLine();
}
private static void UpdateFace()
{
landmarksData = null;
if (faceData == null)
{
return;
}
faceData.Update();
PXCMFaceData.Face face = faceData.QueryFaceByIndex(0);
if (face != null)
{
landmarksData = face.QueryLandmarks();
expressionData = face.QueryExpressions();
poseData = face.QueryPose();
UpdateFaceOrientation(poseData);
UpdateFacePosition(poseData);
UpdateFaceGeometry(landmarksData);
}
}
private int GetFaceExpressionIntensity(PXCMFaceData.ExpressionsData data, FaceExpression faceExpression)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
data.QueryExpression(faceExpression, out score);
return(score.intensity);
}
private bool CheckFaceExpression(PXCMFaceData.ExpressionsData data, FaceExpression faceExpression, int threshold)
{
return(GetFaceExpressionIntensity(data, faceExpression) > threshold);
}
private bool UpdateFace(AnimationTrigger animationTrigger)
{
animationTrigger.IsAnimationDataValid = false;
// Get the closest face
PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);
if (pxcmFace == null)
{
return(false);
}
if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
{
// PXCMFaceData.QueryFaceByIndex(0) failed.
return(false);
}
PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
if (pxcmLandmarksData == null)
{
// PXCMFaceData.Face.QueryLandmarks() failed.
return(false);
}
Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();
PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[]
{
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
};
foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
{
PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);
if (landmarkPoint == null)
{
// PXCMFaceData.LandmarksData.QueryPointIndex() failed.
return(false);
}
landmarkPoints.Add(landmarkType, landmarkPoint);
}
PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
if (pxcmExpressionsData == null)
{
// PXCMFaceData.Face.QueryExpressions() failed.
return(false);
}
animationTrigger.IsAnimationDataValid = true;
PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();
if (pxcmCaptureManager != null)
{
PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
if (pxcmCaptureDevice != null)
{
mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
}
}
animationTrigger.Animations.Clear();
float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;
float leftEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float leftEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
}
else
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
}
// Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
// Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));
PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();
if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
}
else
{
// Error querying expression: EXPRESSION_SMILE.
return(false);
}
return(true);
}
//顔のフレームの更新処理
private void updateFaceFrame()
{
// フレームデータを取得する
PXCMCapture.Sample sample = senceManager.QuerySample();
UpdateColorImage(sample.color);
//SenceManagerモジュールの顔のデータを更新する
faceData.Update();
//検出した顔の数を取得する
int numFaces = faceData.QueryNumberOfDetectedFaces();
//顔のの表出情報のデータ、および角度のデータの入れ物を用意
PXCMFaceData.ExpressionsData[] expressionData = new PXCMFaceData.ExpressionsData[EXPRESSION_MAXFACES];
PXCMFaceData.ExpressionsData.FaceExpressionResult expressionResult;
if (senceManager != null)
{
//それぞれの顔ごとに情報取得および描画処理を行う
for (int i = 0; i < numFaces; ++i)
{
//顔の情報を取得する
PXCMFaceData.Face face = faceData.QueryFaceByIndex(i);
// 顔の位置を取得:Depthで取得する
var detection = face.QueryDetection();
if (detection != null)
{
PXCMRectI32 faceRect;
detection.QueryBoundingRect(out faceRect);
//顔の位置に合わせて長方形を変更
TranslateTransform transform = new TranslateTransform(faceRect.x, faceRect.y);
rect[i].Width = faceRect.w;
rect[i].Height = faceRect.h;
rect[i].Stroke = Brushes.Blue;
rect[i].StrokeThickness = 3;
rect[i].RenderTransform = transform;
//追加：フェイスデータから顔の表情データの情報を得る
expressionData[i] = face.QueryExpressions();
if (expressionData[i] != null)
{
//顔の位置に合わせて姿勢情報を表示
tb[i, 0].RenderTransform = new TranslateTransform(transform.X, transform.Y - 30);
tb[i, 1].RenderTransform = new TranslateTransform(transform.X, transform.Y - 60);
tb[i, 2].RenderTransform = new TranslateTransform(transform.X, transform.Y - 90);
//追加：口の開き具合を取得、表示
if (expressionData[i].QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN, out expressionResult))
//描画処理
{
tb[i, 0].Text = "MOUTH_OPEN:" + expressionResult.intensity;
}
//追加：舌の出し具合を取得、表示
if (expressionData[i].QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_TONGUE_OUT, out expressionResult))
{
//描画処理
tb[i, 1].Text = "TONGUE_OUT:" + expressionResult.intensity;
}
//追加：笑顔の度合を取得、表示
if (expressionData[i].QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out expressionResult))
{
//描画処理
tb[i, 2].Text = "SMILE:" + expressionResult.intensity;
}
}
}
}
}
}
private void updateFaceFrame()
{
PXCMCapture.Sample sample = this.senseManager.QuerySample();
this.image = sample.color;
if (this.image != null)
{
this.invalidate = true;
}
this.faceData.Update();
// get number of faces
FOutFaceLandmarkPoints.SliceCount = 0;
FOutFaceExpressionsResult.SliceCount = 0;
int numFaces = this.faceData.QueryNumberOfDetectedFaces();
for (int i = 0; i < numFaces; ++i)
{
// get faces info
PXCMFaceData.Face face = this.faceData.QueryFaceByIndex(i);
// get face position by Depth
var detection = face.QueryDetection();
if (detection != null)
{
// detection
PXCMRectI32 faceRect;
detection.QueryBoundingRect(out faceRect);
int sliceCount = i + 1;
FOutFacePosition.SliceCount = sliceCount;
FOutFacePosition[i] = new Vector2D(faceRect.x, faceRect.y);
FOutFaceWidth.SliceCount = sliceCount;
FOutFaceWidth[i] = faceRect.w;
FOutFaceHeight.SliceCount = sliceCount;
FOutFaceHeight[i] = faceRect.h;
// pose(only use Depth mode)
PXCMFaceData.PoseData pose = face.QueryPose();
if (pose != null)
{
// faces angle
PXCMFaceData.PoseEulerAngles poseAngle = new PXCMFaceData.PoseEulerAngles();
pose.QueryPoseAngles(out poseAngle);
FOutFacePose.SliceCount = sliceCount;
FOutFacePose[i] = new Vector3D(poseAngle.pitch, poseAngle.yaw, poseAngle.roll);
}
// landmarks
PXCMFaceData.LandmarksData landmarks = face.QueryLandmarks();
FOutFaceLandmarkBinSize.SliceCount = sliceCount;
if (landmarks != null)
{
// number of feature points from landmarks
int numPoints = landmarks.QueryNumPoints();
FOutFaceLandmarkBinSize[i] = numPoints;
PXCMFaceData.LandmarkPoint[] landmarkPoints = new PXCMFaceData.LandmarkPoint[numPoints];
int prevSliceCount = FOutFaceLandmarkPoints.SliceCount;
FOutFaceLandmarkPoints.SliceCount = prevSliceCount + numPoints;
if (landmarks.QueryPoints(out landmarkPoints))
{
for (int j = 0; j < numPoints; j++)
{
int index = prevSliceCount + j;
FOutFaceLandmarkPoints[index] = new Vector2D(landmarkPoints[j].image.x, landmarkPoints[j].image.y);
}
}
}
else
{
FOutFaceLandmarkBinSize[i] = 0;
FOutFaceLandmarkPoints.SliceCount = 0;
}
PXCMFaceData.ExpressionsData expressionData = face.QueryExpressions();
if (expressionData != null)
{
for (int j = 0; j < FInExpressions.SliceCount; j++)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult expressionResult;
if (expressionData.QueryExpression(FInExpressions[j], out expressionResult))
{
FOutFaceExpressionsResult.SliceCount++;
FOutFaceExpressionsResult[j] = expressionResult.intensity;
}
else
{
// do nothing
}
}
}
else
{
FOutFaceExpressionsResult.SliceCount = 0;
}
PXCMFaceData.PulseData pulseData = face.QueryPulse();
if (pulseData != null)
{
FOutPulse.SliceCount = sliceCount;
FOutPulse[i] = pulseData.QueryHeartRate();
}
}
}
}
/// <summary>
/// 面部
/// </summary>
public void FacePipeLine()
{
PXCMSenseManager pp = m_form.Session.CreateSenseManager();
if (pp == null)
{
throw new Exception("PXCMSenseManager null");
}
pp.EnableStream(PXCMCapture.StreamType.STREAM_TYPE_COLOR, 640, 360);
// 面部初始化
pp.EnableFace();
PXCMFaceModule faceModule = pp.QueryFace();
if (faceModule == null)
{
Debug.Assert(true);
return;
}
PXCMFaceConfiguration faceCfg = faceModule.CreateActiveConfiguration();
if (faceCfg == null)
{
Debug.Assert(true);
return;
}
faceCfg.SetTrackingMode(PXCMFaceConfiguration.TrackingModeType.FACE_MODE_COLOR_PLUS_DEPTH);
faceCfg.strategy = PXCMFaceConfiguration.TrackingStrategyType.STRATEGY_CLOSEST_TO_FARTHEST;
// 单个人追踪
faceCfg.detection.maxTrackedFaces = NUM_PERSONS;
faceCfg.landmarks.maxTrackedFaces = NUM_PERSONS;
faceCfg.pose.maxTrackedFaces = NUM_PERSONS;
// 表情初始化
PXCMFaceConfiguration.ExpressionsConfiguration expressionCfg = faceCfg.QueryExpressions();
if (expressionCfg == null)
{
throw new Exception("ExpressionsConfiguration null");
}
expressionCfg.properties.maxTrackedFaces = NUM_PERSONS;
expressionCfg.EnableAllExpressions();
faceCfg.detection.isEnabled = true;
faceCfg.landmarks.isEnabled = true;
faceCfg.pose.isEnabled = true;
if (expressionCfg != null)
{
expressionCfg.Enable();
}
//脉搏初始化
PXCMFaceConfiguration.PulseConfiguration pulseConfiguration = faceCfg.QueryPulse();
if (pulseConfiguration == null)
{
throw new Exception("pulseConfiguration null");
}
pulseConfiguration.properties.maxTrackedFaces = NUM_PERSONS;
if (pulseConfiguration != null)
{
pulseConfiguration.Enable();
}
// 面部识别功能初始化
//PXCMFaceConfiguration.RecognitionConfiguration qrecognition = faceCfg.QueryRecognition();
//if (qrecognition == null)
//{
// throw new Exception("PXCMFaceConfiguration.RecognitionConfiguration null");
//}
//else
//{
// qrecognition.Enable();
//}
faceCfg.ApplyChanges();
if (pp.Init() < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
throw new Exception("Init failed");
}
else
{
using (PXCMFaceData faceData = faceModule.CreateOutput())
{
if (faceData == null)
{
throw new Exception("face data failure");
}
while (!m_form.Stopped)
{
if (pp.AcquireFrame(true).IsError())
{
break;
}
var isConnected = pp.IsConnected();
if (isConnected)
{
var sample = pp.QueryFaceSample();
if (sample == null)
{
pp.ReleaseFrame();
continue;
}
// default is COLOR
DisplayPicture(sample.color);
// 如果检测脸数==0，则continue
faceData.Update();
int nFace = faceData.QueryNumberOfDetectedFaces();
if (nFace == 0)
{
continue;
}
// 存
//PXCMFaceData.Face face = faceData.QueryFaceByIndex(0);
//SaveFaceLandmarkData(face);
//SaveFacialExpressionData(face);
// 获取脸部特征点数据
PXCMFaceData.Face face = faceData.QueryFaceByIndex(0);
Landmarks.updateData(face);
// 获取表情数据
PXCMFaceData.ExpressionsData edata = face.QueryExpressions();
// 多线程加锁，数据同步，与VideoModule会发生竞争
lock (EmotionModel.svmFeature)
{
if (edata != null)
{
// 提取表情数据
int startIdx = EmotionModel.FaceExpressionStartIdx;
for (int i = 0; i < 22; i++)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
edata.QueryExpression((PXCMFaceData.ExpressionsData.FaceExpression)i, out score);
Expression.facialExpressionIndensity[i] = score.intensity;
// 设置SVM Feature
EmotionModel.svmFeature[startIdx + i].Index = startIdx + i;
EmotionModel.svmFeature[startIdx + i].Value = score.intensity;
}
}
// 提取特征点位置数据
int startIdx2 = EmotionModel.FaceLandmarkStartIdx;
for (int i = 0; i < EmotionModel.FaceLandmarkCnt; i++)
{
// 设置SVM Feature
EmotionModel.svmFeature[startIdx2 + i].Index = startIdx2 + i;
EmotionModel.svmFeature[startIdx2 + i].Value = Landmarks.Landmarks[i];
}
}
m_form.UpdatePic();
}
pp.ReleaseFrame();
}
}
}
pp.Close();
pp.Dispose();
}
private void CaptureProcess()
{
Sender = new UDPSender(CurrentIpAdress, Convert.ToInt32(currentPort));
while (senseManager.AcquireFrame(true).IsSuccessful())
{
PXCMCapture.Sample sample = senseManager.QuerySample();
PXCMImage.ImageData colorImageData;
Bitmap colorBitmap;
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorImageData);
colorBitmap = colorImageData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
if (faceModule != null)
{
PXCMFaceData faceData = faceModule.CreateOutput();
faceData.Update();
numberTrackedFace = faceData.QueryNumberOfDetectedFaces();
PXCMFaceData.Face faceDataFace = faceData.QueryFaceByIndex(0);
if (faceDataFace != null)
{
PXCMFaceData.DetectionData faceDetectionData = faceDataFace.QueryDetection();
PXCMFaceData.LandmarksData landMarksData = faceDataFace.QueryLandmarks();
if (faceDetectionData != null) //Запись переменных нахождения фейса
{
PXCMRectI32 faceRectangle;
faceDetectionData.QueryFaceAverageDepth(out faceAverageDepth);
faceDetectionData.QueryBoundingRect(out faceRectangle);
faceRectangleHeight = faceRectangle.h;
faceRectangleWidth = faceRectangle.w;
faceRectangleX = faceRectangle.x;
faceRectangleY = faceRectangle.y;
PXCMFaceData.LandmarkPoint[] points; //Херачим точки на фейс
if (landMarksData != null)
{
bool res = landMarksData.QueryPoints(out points);
Graphics graphics = Graphics.FromImage(colorBitmap);
Font font = new Font(System.Drawing.FontFamily.GenericMonospace, 12, System.Drawing.FontStyle.Bold);
foreach (PXCMFaceData.LandmarkPoint landmark in points)
{
point.X = landmark.image.x + LandMarkAlingment;
point.Y = landmark.image.y + LandMarkAlingment;
//Console.WriteLine(point.X);
if (landmark.confidenceImage == 0)
{
graphics.DrawString("X", font, System.Drawing.Brushes.Brown, point);
}
else
{
graphics.DrawString("*", font, System.Drawing.Brushes.CornflowerBlue, point);
}
Connect = Math.Min(landmark.confidenceImage, 1);
}
}
}
var connectMessage = new SharpOSC.OscMessage("/expressions/connectMessage", Connect);
Sender.Send(connectMessage);
PXCMFaceData.PoseData facePoseData = faceDataFace.QueryPose(); //переменные поворота для анимации головы
if (facePoseData != null)
{
PXCMFaceData.PoseEulerAngles headAngles;
facePoseData.QueryPoseAngles(out headAngles);
headRoll = headAngles.roll;
headYaw = headAngles.yaw;
headPitch = headAngles.pitch;
}
PXCMFaceData.ExpressionsData expressionData = faceDataFace.QueryExpressions();
if (expressionData != null)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_KISS, out score);
Dispatcher.Invoke(() => kissExpression.Text = Convert.ToString(score.intensity));
var kissMessage = new SharpOSC.OscMessage("/expressions/kiss", score.intensity);
Sender.Send(kissMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN, out score);
Dispatcher.Invoke(() => mouthExpression.Text = Convert.ToString(score.intensity));
var mouthMessage = new SharpOSC.OscMessage("/expressions/mouth", score.intensity);
Sender.Send(mouthMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out score);
Dispatcher.Invoke(() => smileExpression.Text = Convert.ToString(score.intensity));
var smileMessage = new SharpOSC.OscMessage("/expressions/smile", score.intensity);
Sender.Send(smileMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_TONGUE_OUT, out score);
Dispatcher.Invoke(() => tongueExpression.Text = Convert.ToString(score.intensity));
var tongueOutMessage = new SharpOSC.OscMessage("/expressions/tongueout", score.intensity);
Sender.Send(tongueOutMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_LOWERER_LEFT, out score);
Dispatcher.Invoke(() => leftBrowLowExpression.Text = Convert.ToString(score.intensity));
var leftBrowLowMessage = new SharpOSC.OscMessage("/expressions/leftBrowLow", score.intensity);
Sender.Send(leftBrowLowMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_LOWERER_RIGHT, out score);
Dispatcher.Invoke(() => rightBrowLowExpression.Text = Convert.ToString(score.intensity));
var rightBrowLowMessage = new SharpOSC.OscMessage("/expressions/rightBrowLow", score.intensity);
Sender.Send(rightBrowLowMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT, out score);
Dispatcher.Invoke(() => leftBrowRaiseExpression.Text = Convert.ToString(score.intensity));
var leftBrowRaiseMessage = new SharpOSC.OscMessage("/expressions/leftBrowRaise", score.intensity);
Sender.Send(leftBrowRaiseMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT, out score);
Dispatcher.Invoke(() => rightBrowRaiseExpression.Text = Convert.ToString(score.intensity));
var rightBrowRaiseMessage = new SharpOSC.OscMessage("/expressions/rightBrowRaise", score.intensity);
Sender.Send(rightBrowRaiseMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT, out score);
Dispatcher.Invoke(() => leftEyeClosedExpression.Text = Convert.ToString(score.intensity));
var leftEyeClosedMessage = new SharpOSC.OscMessage("/expressions/leftEyeClosed", score.intensity);
Sender.Send(leftEyeClosedMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT, out score);
Dispatcher.Invoke(() => rightEyeClosedExpression.Text = Convert.ToString(score.intensity));
var rightEyeClosedMessage = new SharpOSC.OscMessage("/expressions/rightEyeClosed", score.intensity);
Sender.Send(rightEyeClosedMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_TURN_LEFT, out score);
Dispatcher.Invoke(() => eyesTurnLeftExpression.Text = Convert.ToString(score.intensity));
var eyesTurnLeftMessage = new SharpOSC.OscMessage("/expressions/eyesTurnLeft", score.intensity);
Sender.Send(eyesTurnLeftMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_TURN_RIGHT, out score);
Dispatcher.Invoke(() => eyesTurnRightExpression.Text = Convert.ToString(score.intensity));
var eyesTurnRightMessage = new SharpOSC.OscMessage("/expressions/eyesTurnRight", score.intensity);
Sender.Send(eyesTurnRightMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_DOWN, out score);
Dispatcher.Invoke(() => eyesDownExpression.Text = Convert.ToString(score.intensity));
var eyesDownMessage = new SharpOSC.OscMessage("/expressions/eyesDown", score.intensity);
Sender.Send(eyesDownMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_UP, out score);
Dispatcher.Invoke(() => eyesUpExpression.Text = Convert.ToString(score.intensity));
var eyesUpMessage = new SharpOSC.OscMessage("/expressions/eyesUp", score.intensity);
Sender.Send(eyesUpMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_PUFF_LEFT, out score);
Dispatcher.Invoke(() => puffLeftExpression.Text = Convert.ToString(score.intensity));
var leftPuffMessage = new SharpOSC.OscMessage("/expressions/leftPuff", score.intensity);
Sender.Send(leftPuffMessage);
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_PUFF_RIGHT, out score);
Dispatcher.Invoke(() => puffRightExpression.Text = Convert.ToString(score.intensity));
var rightPuffMessage = new SharpOSC.OscMessage("/expressions/rightPuff", score.intensity);
Sender.Send(rightPuffMessage);
}
}
faceData.Dispose();
}
UpdateUI(colorBitmap);
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorImageData);
senseManager.ReleaseFrame();
}
}
//顔のフレームの更新処理
private void updateFaceFrame()
{
// フレームデータを取得する
PXCMCapture.Sample sample = senceManager.QuerySample();
UpdateColorImage(sample.color);
//SenceManagerモジュールの顔のデータを更新する
faceData.Update();
//検出した顔の数を取得する
int numFaces = faceData.QueryNumberOfDetectedFaces();
//顔のの表出情報のデータ、および角度のデータの入れ物を用意
PXCMFaceData.ExpressionsData[] expressionData = new PXCMFaceData.ExpressionsData[EXPRESSION_MAXFACES];
PXCMFaceData.ExpressionsData.FaceExpressionResult expressionResult;
if (senceManager != null)
{
//それぞれの顔ごとに情報取得および描画処理を行う
for (int i = 0; i < numFaces; ++i)
{
//顔の情報を取得する
PXCMFaceData.Face face = faceData.QueryFaceByIndex(i);
// 顔の位置を取得:Depthで取得する
var detection = face.QueryDetection();
if (detection != null)
{
PXCMRectI32 faceRect;
detection.QueryBoundingRect(out faceRect);
//顔の位置に合わせて長方形を変更
TranslateTransform transform = new TranslateTransform(faceRect.x, faceRect.y);
rect[i].Width = faceRect.w;
rect[i].Height = faceRect.h;
rect[i].Stroke = Brushes.Blue;
rect[i].StrokeThickness = 3;
rect[i].RenderTransform = transform;
//追加：フェイスデータから顔の表情データの情報を得る
expressionData[i] = face.QueryExpressions();
if (expressionData[i] != null)
{
//顔の位置に合わせて姿勢情報を表示
tb[i, 0].RenderTransform = new TranslateTransform(transform.X, transform.Y - 30);
tb[i, 1].RenderTransform = new TranslateTransform(transform.X, transform.Y - 60);
tb[i, 2].RenderTransform = new TranslateTransform(transform.X, transform.Y - 90);
//追加：口の開き具合を取得、表示
if (expressionData[i].QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN, out expressionResult))
//描画処理
{
tb[i, 0].Text = "MOUTH_OPEN:" + expressionResult.intensity;
}
//追加：舌の出し具合を取得、表示
if (expressionData[i].QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_TONGUE_OUT, out expressionResult))
{
//描画処理
tb[i, 1].Text = "TONGUE_OUT:" + expressionResult.intensity;
}
//追加：笑顔の度合を取得、表示
if (expressionData[i].QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out expressionResult))
{
//描画処理
tb[i, 2].Text = "SMILE:" + expressionResult.intensity;
}
}
}
}
}
}
private void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// Acquire the color image data
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Get face data
if (faceData != null)
{
faceData.Update();
numFacesDetected = faceData.QueryNumberOfDetectedFaces();
if (numFacesDetected > 0)
{
// Get the first face detected (index 0)
PXCMFaceData.Face face = faceData.QueryFaceByIndex(0);
face.QueryExpressions();
PXCMFaceData.PoseData poseData = face.QueryPose();
// PXCMPoint3DF32 outHeadPosition = new PXCMPoint3DF32(); //F200 has added confidence into struct
PXCMFaceData.HeadPosition outHeadPosition = new PXCMFaceData.HeadPosition();
//processing the head pose data to find the head center position
poseData.QueryHeadPosition(out outHeadPosition);
System.Windows.Media.Media3D.Point3DCollection points = new System.Windows.Media.Media3D.Point3DCollection();
points.Add(new System.Windows.Media.Media3D.Point3D(outHeadPosition.headCenter.x,
outHeadPosition.headCenter.y, outHeadPosition.headCenter.z));
Console.WriteLine("head center position: " + points);
// poseData.QueryHeadPosition(out outHeadPosition);
PXCMFaceData.PoseEulerAngles outPoseEulerAngles = new PXCMFaceData.PoseEulerAngles();
poseData.QueryPoseAngles(out outPoseEulerAngles);
roll = (int)outPoseEulerAngles.roll;
pitch = (int)outPoseEulerAngles.pitch;
yaw = (int)outPoseEulerAngles.yaw;
// PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER what can I do with this?
if (pitch + 12 > 10)
{
headUp = true;
}
else
{
headUp = false;
}
if (pitch < -10)
{
headDown = true;
}
else
{
headDown = false;
}
if (roll > 5)
{
headTiltLeft = true;
}
else
{
headTiltLeft = false;
}
if (roll < -5)
{
headTiltRight = true;
}
else
{
headTiltRight = false;
}
if (yaw > 5)
{
headTurnLeft = true;
}
else
{
headTurnLeft = false;
}
if (yaw < -5)
{
headTurnRight = true;
}
else
{
headTurnRight = false;
}
//Console.WriteLine("Rotation: " + outPoseEulerAngles.roll + " " + outPoseEulerAngles.pitch + " " + outPoseEulerAngles.yaw);
PXCMFaceData.ExpressionsData edata = face.QueryExpressions();
// retrieve the expression information
PXCMFaceData.ExpressionsData.FaceExpressionResult smileScore;
PXCMFaceData.ExpressionsData.FaceExpressionResult eyesUpScore;
PXCMFaceData.ExpressionsData.FaceExpressionResult eyesDownScore;
PXCMFaceData.ExpressionsData.FaceExpressionResult eyesTurnLeftScore;
PXCMFaceData.ExpressionsData.FaceExpressionResult eyesTurnRightScore;
PXCMFaceData.ExpressionsData.FaceExpressionResult headTiltedLeftScore;
PXCMFaceData.ExpressionsData.FaceExpressionResult headTurnedLeftScore;
// PXCMFaceData.ExpressionsData.FaceExpressionResult headUpScore;
//PXCMFaceData.ExpressionsData.FaceExpressionResult headDownScore;
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out smileScore);
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_UP, out eyesUpScore);
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_DOWN, out eyesDownScore);
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_TURN_LEFT, out eyesTurnLeftScore);
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_TURN_RIGHT, out eyesTurnRightScore);
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_HEAD_TILT_LEFT, out headTiltedLeftScore);
edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_HEAD_TURN_LEFT, out headTurnedLeftScore);
// edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_HEAD_UP, out headUpScore);
//edata.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_HEAD_DOWN, out headDownScore);
smile = smileScore.intensity;
eyesUp = eyesUpScore.intensity;
if (eyesUp == 100)
{
eyeIsUp = true;
}
else
{
eyeIsUp = false;
}
eyesDown = eyesDownScore.intensity;
if (eyesDown == 100)
{
eyeIsDown = true;
}
else
{
eyeIsDown = false;
}
eyesTurnLeft = eyesTurnLeftScore.intensity;
eyesTurnRight = eyesTurnRightScore.intensity;
// headTiltLeft = headTiltedLeftScore.intensity;
// headTurnLeft= headTurnedLeftScore.intensity;
// headUp = headUpScore.intensity;
//headDown = headDownScore.intensity;
PXCMCapture.Device device = senseManager.captureManager.device;
device.SetIVCAMAccuracy(PXCMCapture.Device.IVCAMAccuracy.IVCAM_ACCURACY_FINEST);
// eyeIsUP= CheckFaceExpression(edata, FaceExpression.EXPRESSION_EYES_UP, 15);
if ((headTiltLeft | headTurnLeft) & headUp & (eyesTurnLeft == 100) & (!eyeIsDown))
{
looksForward = true;
}
else if ((headTiltRight | headTurnRight) & headUp & (eyesTurnRight == 100) & (!eyeIsDown))
{
looksForward = true;
}
/* else if (headTiltRight & (headDown|headUp) & (!headTurnRight) & (eyesTurnRight==100))
* looksForward = true;
* else if (headTiltLeft & (headDown|headUp) & (!headTurnLeft) & (eyesTurnLeft == 100))
* looksForward = true;
* */
else
{
looksForward = eyeIsUp;
}
// headTiltLeftThreshold = CheckFaceExpression(edata, FaceExpression.EXPRESSION_HEAD_TILT_LEFT, 15);
//csv mona
// var csv = new StringBuilder();
// outputs 10:00 PM
// var newLine = string.Format("{0},{1},{2},{3},{4}{5}", DateTime.Now.ToString("dd-MM-yyyy-hh:mm:ss:fff"), roll, pitch, yaw, eyesUp, Environment.NewLine);
// csv.Append(newLine);
// string pathString = System.IO.Path.Combine(filePath, fileName);
// File.AppendAllText(pathString, csv.ToString());
// Retrieve face location data
PXCMFaceData.DetectionData faceDetectionData = face.QueryDetection();
if (faceDetectionData != null)
{
PXCMRectI32 faceRectangle;
faceDetectionData.QueryBoundingRect(out faceRectangle);
faceRectangleHeight = faceRectangle.h;
faceRectangleWidth = faceRectangle.w;
faceRectangleX = faceRectangle.x;
faceRectangleY = faceRectangle.y;
}
// Process face recognition data
if (face != null)
{
// Retrieve the recognition data instance
recognitionData = face.QueryRecognition();
// Set the user ID and process register/unregister logic
if (recognitionData.IsRegistered())
{
userId = Convert.ToString(recognitionData.QueryUserID());
if (doUnregister)
{
recognitionData.UnregisterUser();
doUnregister = false;
}
}
else
{
if (doRegister)
{
recognitionData.RegisterUser();
// Capture a jpg image of registered user
colorBitmap.Save("image.jpg", System.Drawing.Imaging.ImageFormat.Jpeg);
doRegister = false;
}
else
{
userId = "Unrecognized";
}
}
}
}
else
{
userId = "No users in view";
}
}
// Display the color stream and other UI elements
UpdateUI(colorBitmap);
// Release resources
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
sample.color.Dispose();
// Release the frame
senseManager.ReleaseFrame();
}
}
private void ProcessingThread()
{
// Start AcquireFrame/ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// Acquire the color image data
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
int topScore = 0;
FaceExpression expression = FaceExpression.None;
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
try
{
IBarcodeReader reader = new BarcodeReader();
// load a bitmap
//var barcodeBitmap = (Bitmap)Bitmap.LoadFrom("C:\\sample-barcode-image.png");
// detect and decode the barcode inside the bitmap
var result = reader.Decode(colorBitmap);
// do something with the result
if (result != null)
{
MessageBox.Show(result.BarcodeFormat.ToString());
MessageBox.Show(result.Text);
}
}
catch (Exception ex)
{
}
// Get face data
if (faceData != null)
{
faceData.Update();
numFacesDetected = faceData.QueryNumberOfDetectedFaces();
if (numFacesDetected > 0)
{
// Get the first face detected (index 0)
PXCMFaceData.Face face = faceData.QueryFaceByIndex(0);
// Retrieve face location data
PXCMFaceData.DetectionData faceDetectionData = face.QueryDetection();
if (faceDetectionData != null)
{
PXCMRectI32 faceRectangle;
faceDetectionData.QueryBoundingRect(out faceRectangle);
if ((faceRectangle.h > 90) || (faceRectangle.w > 90))
{
faceRectangleHeight = faceRectangle.h * 3 / 2;
faceRectangleWidth = faceRectangle.w * 3 / 2;
}
else if (((faceRectangle.h < 90) || (faceRectangle.w < 90)) && ((faceRectangle.h > 70) || (faceRectangle.w > 70)))
{
faceRectangleHeight = faceRectangle.h * 2;
faceRectangleWidth = faceRectangle.w * 2;
}
else
{
faceRectangleHeight = faceRectangle.h * 5 / 2;
faceRectangleWidth = faceRectangle.w * 5 / 2;
}
faceRectangleX = faceRectangle.x;
faceRectangleY = faceRectangle.y;
}
// Retrieve pose estimation data
PXCMFaceData.PoseData facePoseData = face.QueryPose();
if (facePoseData != null)
{
PXCMFaceData.PoseEulerAngles headAngles;
facePoseData.QueryPoseAngles(out headAngles);
headRoll = headAngles.roll;
headPitch = headAngles.pitch;
headYaw = headAngles.yaw;
}
// Retrieve expression data
PXCMFaceData.ExpressionsData expressionData = face.QueryExpressions();
if (expressionData != null)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult score;
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_KISS, out score);
expressionScore[Convert.ToInt32(FaceExpression.Kiss)] = score.intensity;
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN, out score);
expressionScore[Convert.ToInt32(FaceExpression.Open)] = score.intensity;
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out score);
expressionScore[Convert.ToInt32(FaceExpression.Smile)] = score.intensity;
expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_TONGUE_OUT, out score);
expressionScore[Convert.ToInt32(FaceExpression.Tongue)] = score.intensity;
// Determine the highest scoring expression
for (int i = 1; i < TotalExpressions; i++)
{
if (expressionScore[i] > topScore)
{
expression = (FaceExpression)i;
}
}
}
// Process face recognition data
if (face != null)
{
// Retrieve the recognition data instance
recognitionData = face.QueryRecognition();
// Set the user ID and process register/unregister logic
if (recognitionData.IsRegistered())
{
userId = Convert.ToString(recognitionData.QueryUserID());
if (doUnregister)
{
recognitionData.UnregisterUser();
SaveDatabaseToFile();
doUnregister = false;
if (_persistentDict.ContainsKey(userId) == true)
{
_persistentDict.Remove(userId);
}
}
}
else
{
if (doRegister)
{
int uId = recognitionData.RegisterUser();
SaveDatabaseToFile();
if (newUserName != "")
{
if (_persistentDict.ContainsKey(uId.ToString()) == false)
{
_persistentDict.Add(uId.ToString(), newUserName);
_persistentDict.Flush();
newUserName = "";
}
}
// Capture a jpg image of registered user
colorBitmap.Save("image.jpg", System.Drawing.Imaging.ImageFormat.Jpeg);
doRegister = false;
}
else
{
userId = "New User";
}
}
}
}
else
{
userId = "No users in view";
}
}
//hand = senseManager.QueryHand();
//if (hand != null)
//{
// // Retrieve the most recent processed data
// handData = hand.CreateOutput();
// handData.Update();
// // Get number of tracked hands
// nhands = handData.QueryNumberOfHands();
// if (nhands > 0)
// {
// // Retrieve hand identifier
// handData.QueryHandId(PXCMHandData.AccessOrderType.ACCESS_ORDER_BY_TIME, 0, out handId);
// // Retrieve hand data
// handData.QueryHandDataById(handId, out ihand);
// PXCMHandData.BodySideType bodySideType = ihand.QueryBodySide();
// if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_LEFT)
// {
// leftHand = true;
// }
// else if (bodySideType == PXCMHandData.BodySideType.BODY_SIDE_RIGHT)
// {
// leftHand = false;
// }
// // Retrieve all hand joint data
// for (int i = 0; i < nhands; i++)
// {
// for (int j = 0; j < 0x20; j++)
// {
// PXCMHandData.JointData jointData;
// ihand.QueryTrackedJoint((PXCMHandData.JointType)j, out jointData);
// nodes[i][j] = jointData;
// }
// }
// // Get world coordinates for tip of middle finger on the first hand in camera range
// handTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.x;
// handTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.y;
// handTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionWorld.z;
// swipehandTipX = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.x;
// swipehandTipY = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.y;
// swipehandTipZ = nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].positionImage.z;
// //Console.Out.WriteLine("Before x={0}", swipehandTipX);
// //Console.Out.WriteLine("Before speed={0}", nodes[0][Convert.ToInt32(PXCMHandData.JointType.JOINT_MIDDLE_TIP)].speed.x);
// // Retrieve gesture data
// if (handData.IsGestureFired("spreadfingers", out gestureData)) { gesture = Gesture.FingerSpread; }
// else if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData)) { gesture = Gesture.Pinch; }
// else if (handData.IsGestureFired("wave", out gestureData)) { gesture = Gesture.Wave; }
// else if (handData.IsGestureFired("swipe_left", out gestureData)) { gesture = Gesture.SwipeLeft; }
// else if (handData.IsGestureFired("swipe_right", out gestureData)) { gesture = Gesture.SwipeRight; }
// else if (handData.IsGestureFired("fist", out gestureData)) { gesture = Gesture.Fist; }
// else if (handData.IsGestureFired("thumb_up", out gestureData)) { gesture = Gesture.Thumb; }
// }
// else
// {
// gesture = Gesture.Undefined;
// }
// //UpdateUI();
// if (handData != null) handData.Dispose();
//}
// Display the color stream and other UI elements
//UpdateUI(colorBitmap, expression, gesture);
UpdateUI(colorBitmap, expression);
// Release resources
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
sample.color.Dispose();
// Release the frame
senseManager.ReleaseFrame();
}
}
public void DrawExpressions(PXCMFaceData.Face face)
{
Debug.Assert(face != null);
// 映像がNULL または 表情認識チェックがOFFの場合処理を終了
if (m_bitmap == null)
{
return;
}
// パラメータ取得
PXCMFaceData.ExpressionsData expressionsOutput = face.QueryExpressions();
// 値を正しく取得出来なかった場合処理を終了
if (expressionsOutput == null)
{
return;
}
lock (m_bitmapLock)
{
using (var brush = new SolidBrush(m_faceTextOrganizer.Colour))
{
const int imageSizeWidth = 18;
const int imageSizeHeight = 18;
/* 座標 */
int positionX = m_faceTextOrganizer.ExpressionsLocation.X;
int positionXText = positionX + imageSizeWidth;
int positionY = m_faceTextOrganizer.ExpressionsLocation.Y;
int positionYText = positionY + imageSizeHeight / 4;
foreach (var expressionEntry in m_expressionDictionary)
{
PXCMFaceData.ExpressionsData.FaceExpression expression = expressionEntry.Key;
PXCMFaceData.ExpressionsData.FaceExpressionResult result;
bool status = expressionsOutput.QueryExpression(expression, out result);
if (!status)
{
continue;
}
//Console.WriteLine("えくすぷれっしょん" + expression);
using (var font = new Font(FontFamily.GenericMonospace, m_faceTextOrganizer.FontSize, FontStyle.Bold))
{
var expressionText = String.Format("= {0}", result.intensity);
//graphics.DrawString(expressionText, font, brush, positionXText, positionYText);
//間隔をあける
smileCnt++;
if (smileCnt > smileN)
{
smileCnt = 0;
//smileである時に処理をする
if (expression.ToString() == "EXPRESSION_SMILE")
{
satisfaction = result.intensity;
RunExpression();
}
}
positionY += imageSizeHeight;
positionYText += imageSizeHeight;
}
}
}
}
}
/// <summary>
/// Update is called every frame, if the MonoBehaviour is enabled.
/// </summary>
void Update()
{
/* Make sure PXCMSenseManager Instance is Initialized */
if (psm == null)
{
return;
}
/* Wait until any frame data is available true(aligned) false(unaligned) */
if (psm.AcquireFrame(true) != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
/* Retrieve face tracking data if ready */
faceAnalyzer = psm.QueryFace();
if (faceAnalyzer != null)
{
PXCMFaceData _outputData = faceAnalyzer.CreateOutput();
if (_outputData != null)
{
_outputData.Update();
for (int i = 0; i < _outputData.QueryNumberOfDetectedFaces(); i++)
{
PXCMFaceData.Face _iFace = _outputData.QueryFaceByIndex(i);
if (_iFace != null)
{
/* Retrieve 78 Landmark Points */
PXCMFaceData.LandmarksData LandmarkData = _iFace.QueryLandmarks();
if (LandmarkData != null)
{
PXCMFaceData.LandmarkPoint[] landmarkPoints = new PXCMFaceData.LandmarkPoint[MaxPoints];
if (LandmarkData.QueryPoints(out landmarkPoints))
{
faceRenderer.DisplayJoints2D(landmarkPoints);
}
}
/* Retrieve Detection Data */
PXCMFaceData.DetectionData detectionData = _iFace.QueryDetection();
if (detectionData != null)
{
PXCMRectI32 rect;
if (detectionData.QueryBoundingRect(out rect))
{
faceRenderer.SetDetectionRect(rect);
}
}
/* Retrieve Pose Data */
PXCMFaceData.PoseData poseData = _iFace.QueryPose();
if (poseData != null)
{
PXCMFaceData.PoseQuaternion poseQuaternion;
if (poseData.QueryPoseQuaternion(out poseQuaternion))
{
faceRenderer.DisplayPoseQuaternion(poseQuaternion);
}
}
/* Retrieve Expression Data */
PXCMFaceData.ExpressionsData expressionData = _iFace.QueryExpressions();
if (expressionData != null)
{
PXCMFaceData.ExpressionsData.FaceExpressionResult expressionResult;
if (expressionData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN, out expressionResult))
{
faceRenderer.DisplayExpression(expressionResult, PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN);
}
}
}
}
/* Retrieve Alert Data */
PXCMFaceData.AlertData _alertData;
for (int i = 0; i < _outputData.QueryFiredAlertsNumber(); i++)
{
if (_outputData.QueryFiredAlertData(i, out _alertData) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
faceRenderer.DisplayAlerts(_alertData);
}
}
}
_outputData.Dispose();
}
/* Retrieve a sample from the camera */
PXCMCapture.Sample sample = psm.QueryFaceSample();
if (sample != null)
{
colorImage = sample.color;
if (colorImage != null)
{
if (colorTexture2D == null)
{
/* If not allocated, allocate a Texture2D */
colorTexture2D = new Texture2D(colorImage.info.width, colorImage.info.height, TextureFormat.ARGB32, false);
/* Associate the Texture2D with a gameObject */
colorPlane.renderer.material.mainTexture = colorTexture2D;
//colorPlane.renderer.material.mainTextureScale = new Vector2(-1f, 1f);
}
/* Retrieve the image data in Texture2D */
PXCMImage.ImageData colorImageData;
colorImage.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorImageData);
colorImageData.ToTexture2D(0, colorTexture2D);
colorImage.ReleaseAccess(colorImageData);
/* Apply the texture to the GameObject to display on */
colorTexture2D.Apply();
}
}
/* Realease the frame to process the next frame */
psm.ReleaseFrame();
}