/// <summary>
/// Selects the FeaturePoints of all tracked skeletons from the source observable.
/// </summary>
/// <param name="observable">The source observable.</param>
/// <param name="faceTracker">The FaceTracker that is used to track the faces.</param>
/// <returns>A sequence of a collection of FeaturePoints and their identifiers in a tuple.</returns>
public static IObservable <IEnumerable <Tuple <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> > > > SelectPersonPoints(this IObservable <Tuple <ColorImageFormat, byte[], DepthImageFormat, short[], Skeleton[]> > observable, FaceTracker faceTracker)
{
if (observable == null)
{
throw new ArgumentNullException("observable");
}
if (faceTracker == null)
{
throw new ArgumentNullException("faceTracker");
}
return(observable.Select(_ => _.Item5.ForEach <Skeleton, Tuple <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> > >(__ =>
{
if (__.TrackingState == SkeletonTrackingState.PositionOnly)
{
return Tuple.Create <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> >(__.TrackingId, __.TrackingState, __.Joints, null);
}
var faceTrackFrame = faceTracker.Track(_.Item1, _.Item2, _.Item3, _.Item4, __);
if (!faceTrackFrame.TrackSuccessful)
{
return Tuple.Create <Int32, SkeletonTrackingState, JointCollection, EnumIndexableCollection <FeaturePoint, PointF> >(__.TrackingId, __.TrackingState, __.Joints, null);
}
return Tuple.Create(__.TrackingId, __.TrackingState, __.Joints, faceTrackFrame.GetProjected3DShape());
})));
}
/// <summary>
/// 顔を認識し回転のパラメータを取得
/// </summary>
/// <param name="colorFrame"></param>
/// <param name="depthFrame"></param>
/// <param name="skeletonFrame"></param>
private void FaceDataAcquisition(ColorImageFrame colorFrame, DepthImageFrame depthFrame, Skeleton skeleton)
{
FaceTrackFrame faceFrame = faceTracker.Track(colorFrame.Format, colorFrame.ToPixelData(),
depthFrame.Format, depthFrame.ToPixelData(), skeleton);
if (faceFrame.TrackSuccessful)
{
rotHeadXYZ.x = faceFrame.Rotation.X;
rotHeadXYZ.y = faceFrame.Rotation.Y;
rotHeadXYZ.z = faceFrame.Rotation.Z;
// トラッキング状態を表示
statusTextBlock.Text = "Status: Tracked";
statusTextBlock.Background = new SolidColorBrush(Color.FromRgb(0xBB, 0xDE, 0xFB));
// 送信
if (sendTextBlock.Text != "push 'Update IP and Port'")
{
SendOSCMessage(rotHeadXYZ, posHeadXYZ, posHandRightXYZ, posHandLeftXYZ);
}
}
else
{
// トラッキング状態を表示
statusTextBlock.Text = "Status: NOT Tracked";
statusTextBlock.Background = (SolidColorBrush)this.FindResource("MyBrush");
}
}
private void SensorAllFrameReady(object sender, AllFramesReadyEventArgs e)
{
using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
{
if (colorFrame != null)
{
colorFrame.CopyPixelDataTo(this.colorPixels);
}
else
{
return;
}
}
using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
{
if (depthFrame != null)
{
depthFrame.CopyPixelDataTo(this.depthPixelsShort);
}
}
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame != null)
{
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
}
}
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton == null)
{
continue;
}
if (skeleton.TrackingState == SkeletonTrackingState.Tracked)
{
_gestureController.Update(skeleton);
faceFrame = faceTracker.Track(this.sensor.ColorStream.Format, this.colorPixels,
this.sensor.DepthStream.Format, this.depthPixelsShort, skeleton);
}
}
colBitmap = new Bitmap(colorWidth, colorHeight);
BitmapData colImageBitmapData = colBitmap.LockBits(new Rectangle(0, 0, colBitmap.Width, colBitmap.Height),
ImageLockMode.WriteOnly,
colBitmap.PixelFormat);
IntPtr IptrColImage = colImageBitmapData.Scan0;
Marshal.Copy(colorPixels, 0, IptrColImage, colorPixels.Length);
colBitmap.UnlockBits(colImageBitmapData);
//显示Bitmap
UpdateColImage(colBitmap);
}
public FaceTrackFrame GetFaceTrackFrame(Skeleton skeleton)
{
if (_faceFrame == null) /* Aus effizienzgruenden wird nicht bei jedem Zugriff ein neues Faceframe erzeugt, sondern nur ein Mal pro Frame. Siehe OnAllFramesReady unten.*/
{
_faceFrame = _faceTracker.Track(_kinectSensor.ColorStream.Format, ColorPixels, _kinectSensor.DepthStream.Format, DepthPixels, skeleton);
}
return(_faceFrame);
}
void TrackFace()
{
var skeleton = skeletonData
.Where(s => s.TrackingState != SkeletonTrackingState.NotTracked)
.OrderBy(s => s.Position.Z)
.FirstOrDefault();
IsSkeletonTrackedUpdated(skeleton != null);
if (skeleton == null)
{
skeletonId = -1;
if (faceTracker != null)
{
faceTracker.Dispose();
faceTracker = null;
}
JawLowerUpdated(null);
return;
}
if (skeletonId != skeleton.TrackingId)
{
try
{
if (faceTracker != null)
{
faceTracker.Dispose();
}
faceTracker = new FaceTracker(sensor);
}
catch (InvalidOperationException)
{
return;
}
}
skeletonId = skeleton.TrackingId;
if (skeleton.TrackingState != SkeletonTrackingState.Tracked)
{
JawLowerUpdated(null);
return;
}
// MEMO: FaceTrackFrame オブジェクトの Dispose メソッドを呼び出すと、以降の処理が正常に続かなくなります。
var faceFrame = faceTracker.Track(sensor.ColorStream.Format, colorImage, sensor.DepthStream.Format, depthImage, skeleton);
if (!faceFrame.TrackSuccessful)
{
JawLowerUpdated(null);
return;
}
var animationUnits = faceFrame.GetAnimationUnitCoefficients();
JawLowerUpdated(animationUnits[AnimationUnit.JawLower]);
}
/// <summary>
/// Selects the successfully tracked FaceTrackFrames of the first tracked skeleton from the AllFramesReadyEventArgs observable.
/// </summary>
/// <param name="source">The source observable</param>
/// <param name="faceTracker">The FaceTracker that is used to track the faces.</param>
/// <returns>A sequence of FaceTrackFrame elements</returns>
public static IObservable <FaceTrackFrame> SelectFaceTrackFrame(this IObservable <AllFramesReadyEventArgs> source, FaceTracker faceTracker)
{
if (source == null)
{
throw new ArgumentNullException("observable");
}
if (faceTracker == null)
{
throw new ArgumentNullException("faceTracker");
}
return(source.SelectFormatStreams()
.Select(_ => faceTracker.Track(_.Item1, _.Item2, _.Item3, _.Item4, _.Item5.First()))
.Where(_ => _.TrackSuccessful));
}
public void Track(Skeleton skeleton)
{
// Colors
if (colors == null)
{
colors = new byte[sensor.ColorStream.FramePixelDataLength];
}
var colorFrame = sensor.ColorStream.OpenNextFrame(0);
if (colorFrame == null)
{
IsLookingToSensor = null;
return;
}
colorFrame.CopyPixelDataTo(colors);
// Depths
if (depths == null)
{
depths = new short[sensor.DepthStream.FramePixelDataLength];
}
var depthFrame = sensor.DepthStream.OpenNextFrame(0);
if (depthFrame == null)
{
IsLookingToSensor = null;
return;
}
depthFrame.CopyPixelDataTo(depths);
// Track
var frame = faceTracker.Track(sensor.ColorStream.Format, colors, sensor.DepthStream.Format, depths, skeleton);
if (frame == null)
{
IsLookingToSensor = null;
return;
}
var shape = frame.Get3DShape();
var leftEyeZ = shape[FeaturePoint.AboveMidUpperLeftEyelid].Z;
var rightEyeZ = shape[FeaturePoint.AboveMidUpperRightEyelid].Z;
IsLookingToSensor = Math.Abs(leftEyeZ - rightEyeZ) <= epsilon;
}
/// <summary>
/// Selects the successfully tracked FaceTrackFrames of all tracked skeletons from the AllFramesReadyEventArgs observable.
/// </summary>
/// <param name="source">The source observable.</param>
/// <param name="faceTracker">The FaceTracker that is used to track the faces.</param>
/// <returns>A sequence of a collection of FaceTrackFrames and their identifier in a tuple.</returns>
public static IObservable <IEnumerable <Tuple <Int32, SkeletonTrackingState, FaceTrackFrame> > > SelectFaceTrackFrame(this IObservable <Tuple <ColorImageFormat, byte[], DepthImageFormat, short[], Skeleton[]> > source, FaceTracker faceTracker)
{
if (source == null)
{
throw new ArgumentNullException("observable");
}
if (faceTracker == null)
{
throw new ArgumentNullException("faceTracker");
}
return(source.Select(_ => _.Item5.Where(__ => __.TrackingState == SkeletonTrackingState.Tracked)
.ForEach(__ => Tuple.Create(__.TrackingId, __.TrackingState, faceTracker.Track(_.Item1, _.Item2, _.Item3, _.Item4, __)))
.Where(__ => __.Item3.TrackSuccessful)));
}
/// <summary>
/// Handles the AllFramesReady event of the kinectSensor control.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="Microsoft.Kinect.AllFramesReadyEventArgs"/> instance containing the event data.</param>
void kinectSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
// Retrieve each single frame and copy the data
using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame())
{
if (colorImageFrame == null)
{
return;
}
colorImageFrame.CopyPixelDataTo(colorPixelData);
}
using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame())
{
if (depthImageFrame == null)
{
return;
}
depthImageFrame.CopyPixelDataTo(depthPixelData);
}
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame == null)
{
return;
}
skeletonFrame.CopySkeletonDataTo(skeletonData);
}
// Retrieve the first tracked skeleton if any. Otherwise, do nothing.
var skeleton = skeletonData.FirstOrDefault(s => s.TrackingState == SkeletonTrackingState.Tracked);
if (skeleton == null)
{
return;
}
// Make the faceTracker processing the data.
FaceTrackFrame faceFrame = faceTracker.Track(kinectSensor.ColorStream.Format, colorPixelData,
kinectSensor.DepthStream.Format, depthPixelData,
skeleton);
// If a face is tracked, then we can use it
if (faceFrame.TrackSuccessful)
{
// Retrieve only the Animation Units coeffs
var AUCoeff = faceFrame.GetAnimationUnitCoefficients();
// Records to list buffer if record is enabled
if (isRecord == true)
{
// Start stopwatch
stopwatch.Start();
// AU coefficients
lipRaiserBuffer.Add(AUCoeff[AnimationUnit.LipRaiser]);
jawLowerBuffer.Add(AUCoeff[AnimationUnit.JawLower]);
lipStretchBuffer.Add(AUCoeff[AnimationUnit.LipStretcher]);
browLowerBuffer.Add(AUCoeff[AnimationUnit.BrowLower]);
lipDepressBuffer.Add(AUCoeff[AnimationUnit.LipCornerDepressor]);
browRaiserBuffer.Add(AUCoeff[AnimationUnit.BrowLower]);
// Face rotation
xRotation.Add(faceFrame.Rotation.X);
yRotation.Add(faceFrame.Rotation.Y);
zRotation.Add(faceFrame.Rotation.Z);
// Get time in ms
timeBuffer.Add(stopwatch.ElapsedMilliseconds);
}
// Display on UI coefficients and rotation for user
LipRaiser.Content = AUCoeff[AnimationUnit.LipRaiser];
JawLower.Content = AUCoeff[AnimationUnit.JawLower];
LipStretch.Content = AUCoeff[AnimationUnit.LipStretcher];
BrowLower.Content = AUCoeff[AnimationUnit.BrowLower];
LipDepress.Content = AUCoeff[AnimationUnit.LipCornerDepressor];
BrowRaiser.Content = AUCoeff[AnimationUnit.BrowRaiser];
XRotation.Content = faceFrame.Rotation.X;
YRotation.Content = faceFrame.Rotation.Y;
ZRotation.Content = faceFrame.Rotation.Z;
// Animates the drawn face
var jawLowerer = AUCoeff[AnimationUnit.JawLower];
jawLowerer = jawLowerer < 0 ? 0 : jawLowerer;
MouthScaleTransform.ScaleY = jawLowerer * 5 + 0.1;
MouthScaleTransform.ScaleX = (AUCoeff[AnimationUnit.LipStretcher] + 1);
LeftBrow.Y = RightBrow.Y = (AUCoeff[AnimationUnit.BrowLower]) * 40;
RightBrowRotate.Angle = (AUCoeff[AnimationUnit.BrowRaiser] * 20);
LeftBrowRotate.Angle = -RightBrowRotate.Angle;
CanvasRotate.Angle = faceFrame.Rotation.Z;
}
}
void newSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
if (_faceTracker == null)
{
try
{
_faceTracker = new FaceTracker(_sensorChooser.Kinect);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
_faceTracker = null;
}
}
#region copying color data
using (var colorFrame = e.OpenColorImageFrame())
{
if (colorFrame == null)
{
return;
}
// Make a copy of the color frame for displaying.
var haveNewFormat = _currentColorImageFormat != colorFrame.Format;
if (haveNewFormat)
{
_currentColorImageFormat = colorFrame.Format;
_colorImageData = new byte[colorFrame.PixelDataLength];
_colorImageWritableBitmap = new WriteableBitmap(colorFrame.Width, colorFrame.Height, 96, 96, PixelFormats.Bgr32, null);
_colorImage.Source = _colorImageWritableBitmap;
}
colorFrame.CopyPixelDataTo(_colorImageData);
_colorImageWritableBitmap.WritePixels(
new Int32Rect(0, 0, colorFrame.Width, colorFrame.Height),
_colorImageData,
colorFrame.Width * colorFrame.BytesPerPixel,
0);
}
#endregion
#region copying depth data
using (var depthFrame = e.OpenDepthImageFrame())
{
if (depthFrame == null)
{
return;
}
depthData = new short[depthFrame.PixelDataLength];
depthFrame.CopyPixelDataTo(depthData);
}
#endregion
#region copying skeleton data
using (var skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame == null)
{
return;
}
Skeleton[] allSkeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
skeletonFrame.CopySkeletonDataTo(allSkeletons);
firstSkeleton = (from c in allSkeletons
where c.TrackingState == SkeletonTrackingState.Tracked
select c).FirstOrDefault();
if (firstSkeleton == null)
{
return;
}
}
#endregion
if (_faceTracker != null)
{
FaceTrackFrame frame = _faceTracker.Track(_sensorChooser.Kinect.ColorStream.Format,
_colorImageData, _sensorChooser.Kinect.DepthStream.Format, depthData, firstSkeleton);
if (frame.TrackSuccessful)
{
AnalyzeFace(frame);
if (_thoughtBubble.Visibility != System.Windows.Visibility.Visible)
{
_thoughtBubble.Visibility = System.Windows.Visibility.Visible;
}
}
else
{
}
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
//redimensionar o array dos esqueletos, se necessario
if (this.skeletons.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
skeletonFrame.CopySkeletonDataTo(this.skeletons);
if (this.colorPixelData == null || colorPixelData.Length != colorImageFrame.PixelDataLength)
{
this.colorPixelData = new byte[colorImageFrame.PixelDataLength];
}
colorImageFrame.CopyPixelDataTo(this.colorPixelData);
//detectar o numero de utilizadores e qual o mais proximo
var newNearestId = -1;
var nearestDistance2 = double.MaxValue;
int nUsers = 0;
int nTrackedUsers = 0;
foreach (var skeleton in skeletons)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked)
{
// Find the distance squared.
var distance2 = (skeleton.Position.X * skeleton.Position.X) +
(skeleton.Position.Y * skeleton.Position.Y) +
(skeleton.Position.Z * skeleton.Position.Z);
// Is the new distance squared closer than the nearest so far?
if (distance2 < nearestDistance2)
{
newNearestId = skeleton.TrackingId;
nearestDistance2 = distance2;
}
nTrackedUsers++;
}
if (skeleton.TrackingState != SkeletonTrackingState.NotTracked)
{
nUsers++;
}
}
this.nearestId = newNearestId;
this.NumberOfDetectUsers = nUsers;
//processar a detecao de movimento
if (detectMotion && motionDetector != null)
{
if (this.depthPixelData == null || depthPixelData.Length != depthImageFrame.BytesPerPixel)
{
this.depthPixelData = new short[depthImageFrame.PixelDataLength];
}
depthImageFrame.CopyPixelDataTo(depthPixelData);
var coloredPixels = new byte[colorImageFrame.PixelDataLength];
colorImageFrame.CopyPixelDataTo(coloredPixels);
var colorBitmap = ImageProcessingAux.CreateBitmapFromPixelData(coloredPixels, colorImageFrame.Width, colorImageFrame.Height);
var playerBitmap = ImageProcessingAux.ProccessPlayerPixels(coloredPixels, depthPixelData, colorImageFrame.Format);
motionDetector.ProcessFrame(ref colorBitmap, ref playerBitmap);
}
//processar a deteccao da face
if (trackFace && faceTracker != null)
{
faceTracker.Track(skeletons, colorImageFrame, depthImageFrame, nearestId);
}
//processar gestos
if (gestureDetectorsValue.Count > 0)
{
foreach (Skeleton s in skeletons)
{
if (s.TrackingId == nearestId)
{
foreach (GestureDetector gd in gestureDetectorsValue)
{
gd.Add(s.Joints[gd.TrackedJoint].Position, kinectSensorValue);
}
}
}
}
//processar posturas
if (postureDetectorsValue.Count > 0)
{
foreach (Skeleton s in skeletons)
{
if (s.TrackingId == nearestId)
{
foreach (PostureDetector pd in postureDetectorsValue)
{
pd.Add(s, kinectSensorValue);
}
}
}
}
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
File.AppendAllText("mouseLog.txt", DateTime.Now + " - All Kinect frames ready.\n");
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Color- depth or Skeletonframe is null. Aborting Frame.\n");
return;
}
else //if (status == StatusEnum.ReadyActivated || status == StatusEnum.Initializing || status == StatusEnum.UnreadyDeactivated)
{
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
HandlePossibleImageFormatChanges(colorImageFrame, depthImageFrame);
WriteDataToMembers(colorImageFrame, depthImageFrame, skeletonFrame);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
if (activeSkeleton != null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Skeleton is there. Trying to find face.\n");
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
LogFaceDetection(currentFaceFrame);
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
Bitmap leftEye2 = null;
Bitmap rightEye2 = null;
if (leftEye != null)
{
leftEye2 = new Bitmap(leftEye);
}
if (rightEye != null)
{
rightEye2 = new Bitmap(rightEye);
}
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbLeft.Image = rightEye2));
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye2));
//Wende Kantenfilter auf die beiden Augen an.
if (rightEye != null && leftEye != null)
{
Dictionary <string, int> angleCount;
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out angleCount);
bool rightEyeClosed = IsEyeClosed(angleCount);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out angleCount);
bool leftEyeClosed = IsEyeClosed(angleCount);
if (rightEyeClosedHistory.Count > 100)
{
rightEyeClosedHistory.RemoveAt(0);
}
if (leftEyeClosedHistory.Count > 100)
{
leftEyeClosedHistory.RemoveAt(0);
}
leftEyeClosedHistory.Add(leftEyeClosed);
rightEyeClosedHistory.Add(rightEyeClosed);
//If Face is rotated, move Mouse
MoveMouseAccordingToFaceRotation(currentFaceFrame);
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face recognized but couldn't find eye in face.\n");
}
clickDelay++;
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Active Skeleton is null. Couldn't analyze frame.\n");
}
}
}
catch (Exception e)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Error during frame analyzation.\n" + e.ToString());
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
/// <summary>
/// Handles the AllFramesReady event of the kinectSensor control.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="Microsoft.Kinect.AllFramesReadyEventArgs"/> instance containing the event data.</param>
void kinectSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
// Retrieve each single frame and copy the data
using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame())
{
if (colorImageFrame == null)
{
return;
}
colorImageFrame.CopyPixelDataTo(colorPixelData);
}
using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame())
{
if (depthImageFrame == null)
{
return;
}
depthImageFrame.CopyPixelDataTo(depthPixelData);
}
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame == null)
{
return;
}
skeletonFrame.CopySkeletonDataTo(skeletonData);
}
// Retrieve the first tracked skeleton if any. Otherwise, do nothing.
var skeleton = skeletonData.FirstOrDefault(s => s.TrackingState == SkeletonTrackingState.Tracked);
if (skeleton == null)
{
return;
}
// Make the faceTracker processing the data.
FaceTrackFrame faceFrame = faceTracker.Track(kinectSensor.ColorStream.Format, colorPixelData,
kinectSensor.DepthStream.Format, depthPixelData,
skeleton);
// If a face is tracked, then we can use it.
if (faceFrame.TrackSuccessful)
{
var triangles = faceFrame.GetTriangles();
// Retrieve only the Animation Units coeffs.
var AUCoeff = faceFrame.GetAnimationUnitCoefficients();
var jawLowerer = AUCoeff[AnimationUnit.JawLower];
jawLowerer = jawLowerer < 0 ? 0 : jawLowerer;
MouthScaleTransform.ScaleY = jawLowerer * 5 + 0.1;
MouthScaleTransform.ScaleX = (AUCoeff[AnimationUnit.LipStretcher] + 1);
LeftBrow.Y = RightBrow.Y = (AUCoeff[AnimationUnit.BrowLower]) * 40;
RightBrowRotate.Angle = (AUCoeff[AnimationUnit.BrowRaiser] * 20);
LeftBrowRotate.Angle = -RightBrowRotate.Angle;
CanvasRotate.Angle = -faceFrame.Rotation.Z;
// CanvasTranslate.X = faceFrame.Translation.X;
// CanvasTranslate.Y = faceFrame.Translation.Y;
if (logToFile)
{
writeToFile(filename_txt.Text, faceFrame);
}
if (writeToOSC)
{
sendOsc(osc_channel_txt.Text, faceFrame, oscWriter);
}
}
}
private void OnAllFramesReady(object sender, Microsoft.Kinect.AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.SkeletonData);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
//Idea: Separate Eye-Parts of Color Image
//Use learning Algorithm for right and left eye
//Detect blink on separated parts of color Image
//colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values
if (activeSkeleton != null)
{
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser];
float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower];
tbBrowLowerer.Text = browLowererValue.ToString();
tbBrowRaiser.Text = browRaiserValue.ToString();
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
pbLeftEye.Image = leftEye;
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
pbRightEye.Image = rightEye;
//Wende Kantenfilter auf die beiden Augen an.
double dxRight;
double dyRight;
double dxLeft;
double dyLeft;
if (rightEye != null && leftEye != null)
{
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out dxRight, out dyRight);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out dxLeft, out dyLeft);
//If Face is rotated, move Mouse
if (headRotationHistory.Count > filterLength && currentFaceFrame.TrackSuccessful)
{
int x = 0;
int y = 0;
//Method 1: Ohne Glättung
//ScaleXY(currentFaceFrame.Rotation, out x, out y);
//MouseControl.Move(x, y);
////Method 2: Glättung über die letzten x Bilder:
//int i = 0;
//Vector3DF rotationMedium = new Vector3DF();
//while (i < 10 && headRotationHistory.Count - 1 > i)
//{
// i++;
// rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X;
// rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y;
//}
//rotationMedium.X = rotationMedium.X / i;
//rotationMedium.Y = rotationMedium.Y / i;
//ScaleXY(rotationMedium, out x, out y);
//MouseControl.Move(x, y);
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < filterLength - 1)
{
i++;
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
ScaleXY(rotationMedium, out x, out y);
MouseControl.Move(x, y);
//Method 4: Quadratische Glättung
//double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y));
//double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X));
//if (deltaX < 0)
// deltaX = -Math.Pow(deltaX, 2) * 4;
//else
// deltaX = Math.Pow(deltaX, 2) * 4;
//if (deltaY < 0)
// deltaY = -Math.Pow(deltaY, 2) * 5;
//else
// deltaY = Math.Pow(deltaY, 2) * 5;
//MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY));
}
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
}
}
catch (Exception e)
{
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
void kinectSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
// Retrieve each single frame and copy the data
using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame())
{
if (colorImageFrame == null)
{
return;
}
colorImageFrame.CopyPixelDataTo(colorPixelData);
//int strade = colorImageFrame.Width * 4;
//image1.Source = BitmapSource.Create(colorImageFrame.Width, colorImageFrame.Height, 96, 96,
// PixelFormats.Bgr32, null, colorPixelData, strade);
}
using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame())
{
if (depthImageFrame == null)
{
return;
}
depthImageFrame.CopyPixelDataTo(depthPixelData);
}
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame == null)
{
return;
}
skeletonFrame.CopySkeletonDataTo(skeletonData);
}
// Retrieve the first tracked skeleton if any. Otherwise, do nothing.
var skeleton = skeletonData.FirstOrDefault(s => s.TrackingState == SkeletonTrackingState.Tracked);
if (skeleton == null && !sessionClose)
{
serialP.WriteLine("s");
serialP.WriteLine("c");
serialP.WriteLine("p");
serialP.WriteLine("g");
if (isActive)
{
isActive = false;
}
slejenie = false;
activatorRightHand = 0;
activatorLeftHand = false;
firstMeet = false;
sessionClose = true;
return;
}
else if (skeleton != null && !firstMeet)
{
serialP.WriteLine("i");
playsound(comms[0]);
firstMeet = true;
sessionClose = false;
}
if (sessionClose)
{
return;
}
// Make the faceTracker processing the data.
FaceTrackFrame faceFrame = faceTracker.Track(kinectSensor.ColorStream.Format, colorPixelData,
kinectSensor.DepthStream.Format, depthPixelData,
skeleton);
EnumIndexableCollection <FeaturePoint, PointF> facePoints = faceFrame.GetProjected3DShape();
// points of hands and shoulder - to determine HELLO, etc.
Joint shoulderCenter = skeleton.Joints[JointType.ShoulderCenter];
Joint head = skeleton.Joints[JointType.Head];
Joint rightHand = skeleton.Joints[JointType.HandRight];
Joint leftHand = skeleton.Joints[JointType.HandLeft];
// initialize sound for hello
//SoundPlayer a = new SoundPlayer("C:\\sal.wav");
// open stream for uart reading
//serialP.Open();
// points of lip's corner - with help of this I determine smile
double x1 = facePoints[88].X;
double y1 = facePoints[88].Y;
System.Windows.Point leftLip = new System.Windows.Point(x1, y1);
double x2 = facePoints[89].X;
double y2 = facePoints[89].Y;
System.Windows.Point rightLip = new System.Windows.Point(x2, y2);
Vector subtr = System.Windows.Point.Subtract(leftLip, rightLip);
// distance between kinect and human
distance = skeleton.Position.Z * 100;
// distance between two corners of lip
double length = Math.Sqrt(subtr.X * subtr.X + subtr.Y * subtr.Y);
int check = 100;
double angle1 = 0d;
double angle2 = 0d;
double angle = skeleton.Position.X * 100;
#region "Smile deterine"
if (distance >= 95 && distance < 110)
{
check = 22;
}
else if (distance >= 110 && distance < 120)
{
check = 19;
}
else if (distance >= 120 && distance < 130)
{
check = 18;
}
else if (distance >= 130 && distance < 140)
{
check = 17;
}
else if (distance >= 140 && distance < 150)
{
check = 16;
}
else if (distance >= 150 && distance < 160)
{
check = 14;
}
else if (distance >= 160 && distance < 170)
{
check = 13;
}
else if (distance >= 170 && distance < 180)
{
check = 12;
}
else if (distance >= 180 && distance < 190)
{
check = 11;
}
#endregion
#region "Angle"
if (distance >= 90 && distance < 110)
{
angle1 = -15;
angle2 = 15;
}
else if (distance >= 110 && distance < 150)
{
angle1 = -20;
angle2 = 20;
}
else if (distance >= 150 && distance < 170)
{
angle1 = -30;
angle2 = 30;
}
else if (distance >= 170 && distance < 200)
{
angle1 = -35;
angle2 = 35;
}
else if (distance >= 200)
{
angle1 = -40;
angle2 = 40;
}
#endregion
double condition1 = Math.Abs(leftHand.Position.Z * 100 - shoulderCenter.Position.Z * 100);
double condition2 = Math.Abs(rightHand.Position.Z * 100 - shoulderCenter.Position.Z * 100);
// If position of two hands higher than shoulder it's activate 'slejenie za ob'ektom'
if (condition1 > 45 &&
condition2 > 45 &&
leftHand.Position.X < rightHand.Position.X)
{
if (!slejenie)
{
isActive = true;
FIXED_DISTANCE = distance;
slejenie = true;
}
}
// The command to stop 'slejenie za ob'ektom'
if (leftHand.Position.X > rightHand.Position.X)
{
isActive = false;
}
// Slejenie za ob'ektom
if (isActive)
{
int pinkIs = (int)typeCondition.THIRD;
int purpleIs = (int)typeCondition.FORTH;
int redIs = (int)typeCondition.FIVTH;
int yellowIs = (int)typeCondition.SIXTH;
if (distance > FIXED_DISTANCE + 10.0d)
{
if (angle < angle1)
{
ellipseSmile.Fill = Brushes.Pink;
if (currentAction != pinkIs)//povorot na pravo
{
currentAction = pinkIs;
serialP.WriteLine("r");
}
}
else if (angle > angle2)//povorot na levo
{
ellipseSmile.Fill = Brushes.Purple;
if (currentAction != purpleIs)
{
currentAction = purpleIs;
serialP.WriteLine("l");
}
}
else
{
ellipseSmile.Fill = Brushes.Red;
if (currentAction != redIs)// vpered
{
currentAction = redIs;
serialP.WriteLine("f");
}
}
}
else if (distance > 90)
{
if (angle < angle1)
{
ellipseSmile.Fill = Brushes.Pink;
if (currentAction != pinkIs)//na pravo
{
currentAction = pinkIs;
serialP.WriteLine("r");
}
}
else if (angle > angle2)
{
ellipseSmile.Fill = Brushes.Purple;
if (currentAction != purpleIs)// na levo
{
currentAction = purpleIs;
serialP.WriteLine("l");
}
}
else
{
ellipseSmile.Fill = Brushes.Yellow;
if (currentAction != yellowIs)//stop, ili - do nothing
{
currentAction = yellowIs;
serialP.WriteLine("s");
}
}
}
else
{
ellipseSmile.Fill = Brushes.Yellow;
if (currentAction != yellowIs)//stop, ili - do nothing
{
currentAction = yellowIs;
serialP.WriteLine("s");
}
}
}
// esli 'slejenie za ob'ektom' otklu4en
else if (!isActive)
{
int blueIs = (int)typeCondition.FIRST;
int blackIs = (int)typeCondition.SECOND;
int onkol = (int)typeCondition.SEVENTH;
if (leftHand.Position.Y > head.Position.Y && rightHand.Position.Y < shoulderCenter.Position.Y)
{
ellipseSmile.Fill = Brushes.Blue;
if (currentAction != blueIs && !activatorLeftHand)//privet levoi rukoi ----------------------------------------------------------------------------
{
currentAction = blueIs;
serialP.WriteLine("q");
activatorLeftHand = true;
}
}
else if (rightHand.Position.Y > head.Position.Y && leftHand.Position.Y < shoulderCenter.Position.Y)
{
ellipseSmile.Fill = Brushes.Blue;
if (currentAction != onkol && activatorRightHand != 12)//privet pravoi rukoi -----------------------------------------------------------------------------
{
currentAction = onkol;
serialP.WriteLine("w");
activatorRightHand = 12;
}
}
else
{
ellipseSmile.Fill = Brushes.Black;
if (currentAction != blackIs)// toktaidy ili do nothing
{
currentAction = blackIs;
serialP.WriteLine("s");
}
if (currentAction == blackIs)
{
if (length >= check && currentFace != (int)faceConditions.FIRST)
{
serialP.WriteLine("z"); // smile
currentFace = (int)faceConditions.FIRST;
ellipseSmile.Fill = Brushes.Brown;
}
else if (length < check && currentFace != (int)faceConditions.SECOND)
{
serialP.WriteLine("x"); // poker face
currentFace = (int)faceConditions.SECOND;
ellipseSmile.Fill = Brushes.Gold;
}
#region "povoroti golovoi"
if (angle < angle1)
{
ellipseSmile.Fill = Brushes.Pink;
if (!headToRight)//povorot golovi na pravo
{
headToRight = true;
headToCenter = false;
headToLeft = false;
serialP.WriteLine("k");
}
}
else if (angle > angle2)//povorot golovi na levo
{
if (!headToLeft)
{
headToLeft = true;
headToCenter = false;
headToRight = false;
serialP.WriteLine("j");
}
}
else if (angle < angle2 && angle > angle1)//golova v centre
{
if (!headToCenter)
{
headToCenter = true;
headToRight = false;
headToLeft = false;
serialP.WriteLine("p");
}
}
#endregion
}
else if (!faceFrame.TrackSuccessful && currentFace != (int)faceConditions.NONE)
{
serialP.WriteLine("c"); // sad face
currentFace = (int)faceConditions.NONE;
ellipseSmile.Fill = Brushes.Chocolate;
}
}
}
label2.Content = distance.ToString();
//label1.Content = (leftHand.Position.Z * 100).ToString();
//label3.Content = (shoulderCenter.Position.Z * 100).ToString();
//serialP.Close();
}
public AppModel()
{
KinectManager.SensorConnected
.Subscribe(sensor =>
{
sensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
sensor.DepthStream.Enable(DepthImageFormat.Resolution320x240Fps30);
sensor.SkeletonStream.EnableWithDefaultSmoothing();
try
{
sensor.Start();
}
catch (Exception ex)
{
// センサーが他のプロセスに既に使用されている場合に発生します。
Debug.WriteLine(ex);
}
});
KinectManager.SensorDisconnected
.Subscribe(sensor => sensor.Stop());
KinectManager.Initialize();
// フレームのデータ。
var frameData = Observable.Interval(FramesInterval)
.Select(_ => new
{
Sensor = KinectManager.Sensor.Value,
ColorData = KinectManager.Sensor.Value.GetColorData(FramesInterval),
DepthData = KinectManager.Sensor.Value.GetDepthDataInInt16(FramesInterval),
BodyData = KinectManager.Sensor.Value.GetSkeletonData(FramesInterval),
})
.ToReadOnlyReactiveProperty(null, ReactivePropertyMode.DistinctUntilChanged);
// ターゲットの人物。
TargetBody = frameData
.Where(_ => _.BodyData != null)
.Select(_ => GetTargetBody(_.BodyData, TargetBody.Value))
.ToReadOnlyReactiveProperty(null, ReactivePropertyMode.DistinctUntilChanged);
// ターゲットの人物が存在するかどうか。
HasTargetBody = TargetBody
.Select(b => b != null)
.ToReadOnlyReactiveProperty();
// ヒット ゾーン内の手。z 要素は体からの差分。
HitHand = TargetBody
.Select(b => GetHitHand(b, HitHand.Value))
.ToReadOnlyReactiveProperty(null, ReactivePropertyMode.DistinctUntilChanged);
// ヒット ゾーン内かどうか。
IsHandHit = HitHand
.Select(_ => _.HasValue)
.ToReadOnlyReactiveProperty();
AreHandsAbove = TargetBody.Select(GetAreHandsAbove).ToReadOnlyReactiveProperty();
IsSquat = TargetBody.Select(GetIsSquat).ToReadOnlyReactiveProperty();
IsJumping = TargetBody.Select(GetIsJumping).ToReadOnlyReactiveProperty();
BodyOrientation = TargetBody.Select(GetBodyOrientation).ToReadOnlyReactiveProperty();
IsLeftOriented = BodyOrientation.Select(x => x < -0.35).ToReadOnlyReactiveProperty();
IsRightOriented = BodyOrientation.Select(x => x > 0.35).ToReadOnlyReactiveProperty();
JawLower = TargetBody
.Select(body =>
{
if (body == null)
{
return(0);
}
var data = frameData.Value;
if (data.ColorData == null || data.DepthData == null || data.BodyData == null)
{
return(0);
}
if (faceTracker == null)
{
faceTracker = new FaceTracker(data.Sensor);
}
var faceFrame = faceTracker.Track(data.Sensor.ColorStream.Format, data.ColorData, data.Sensor.DepthStream.Format, data.DepthData, body);
if (!faceFrame.TrackSuccessful)
{
return(0);
}
var animationUnits = faceFrame.GetAnimationUnitCoefficients();
return(animationUnits[AnimationUnit.JawLower]);
})
.ToReadOnlyReactiveProperty();
IsMouthOpen = JawLower.Select(x => x > 0.35).ToReadOnlyReactiveProperty();
}
private async Task FaceTrackingAsync(TimeSpan dueTime, TimeSpan interval, CancellationToken token)
{
if (interval.TotalMilliseconds == 0)
{
return;
}
// Initial wait time before we begin the periodic loop.
if (dueTime > TimeSpan.Zero)
{
await Task.Delay(dueTime, token);
}
DateTime LocalTimestamp = Timestamp;
FaceTracker tracker = new FaceTracker(Sensor);
// Repeat this loop until cancelled.
while (!token.IsCancellationRequested)
{
// Skip already work with given data
if (Timestamp == LocalTimestamp)
{
await Task.Delay(interval, token);
continue;
}
// Timestamp data
LocalTimestamp = Timestamp;
FaceTrackWatch.Again();
// Do Job
try {
CopyColorData = true;
CopySkeletons = true;
FPoints = null;
Mood = 0;
if (null != GestureManager && null != GestureManager.Skeleton)
{
FaceTrackFrame frame = tracker.Track(ColorFormat, ColorData, DepthFormat, DepthData, GestureManager.Skeleton);
if (frame.TrackSuccessful)
{
// Only once. It doesn't change.
if (FTriangles == null)
{
FTriangles = frame.GetTriangles();
}
FPoints = frame.GetProjected3DShape();
Mood = frame.GetAnimationUnitCoefficients()[AnimationUnit.LipCornerDepressor];
WSRProfileManager.GetInstance().UpdateMood(Mood);
}
}
}
catch (Exception ex) {
WSRConfig.GetInstance().logError("FACE", ex);
}
FaceTrackWatch.Stop();
// Wait to repeat again.
if (interval > TimeSpan.Zero)
{
await Task.Delay(interval, token);
}
}
// Dispose Tracker
tracker.Dispose();
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
File.AppendAllText("mouseLog.txt", DateTime.Now + " - All Kinect frames ready.\n");
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Color- depth or Skeletonframe is null. Aborting Frame.\n");
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.SkeletonData);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
//Idea: Separate Eye-Parts of Color Image
//Use learning Algorithm for right and left eye
//Detect blink on separated parts of color Image
//colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values
if (activeSkeleton != null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Skeleton is there. Trying to find face.\n");
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
if (currentFaceFrame.TrackSuccessful)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Recognized face successfully.\n");
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Couldn't find face in frame.\n");
}
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
//this.pbRight.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye));
//
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
Bitmap leftEye2 = null;
Bitmap rightEye2 = null;
if (leftEye != null)
{
leftEye2 = new Bitmap(leftEye);
}
if (rightEye != null)
{
rightEye2 = new Bitmap(rightEye);
}
// System.Delegate d = new MethodInvoker(SetPictures));
// this.Invoke(SetPictures, leftEye);
//pbRight.Image = rightEye;
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbLeft.Image = rightEye2));
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye2));
// this.Invoke(new MethodInvoker(SetPictures));
//Wende Kantenfilter auf die beiden Augen an.
if (rightEye != null && leftEye != null)
{
Dictionary <string, int> angleCount;
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out angleCount);
bool rightEyeClosed = IsEyeClosed(angleCount);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out angleCount);
bool leftEyeClosed = IsEyeClosed(angleCount);
// pbLeftFaltung.Image = edgePicLeft;
// pbRightFaltung.Image = edgePicRight;
if (rightEyeClosedHistory.Count > 100)
{
rightEyeClosedHistory.RemoveAt(0);
}
if (leftEyeClosedHistory.Count > 100)
{
leftEyeClosedHistory.RemoveAt(0);
}
leftEyeClosedHistory.Add(leftEyeClosed);
rightEyeClosedHistory.Add(rightEyeClosed);
//If Face is rotated, move Mouse
if (headRotationHistory.Count > gaussFilter.Count - 1 && leftEyeClosedHistory.Count > nudConvolutionFilterLength.Value && currentFaceFrame.TrackSuccessful)
{
int x = 0;
int y = 0;
float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser];
float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower];
float mouthOpenValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.JawLower];
if (browRaiserHistory.Count >= 100)
{
browRaiserHistory.RemoveAt(0);
browLowererHistory.RemoveAt(0);
mouthOpenHistory.RemoveAt(0);
}
browLowererHistory.Add(browLowererValue);
browRaiserHistory.Add(browRaiserValue);
mouthOpenHistory.Add(mouthOpenValue);
//Method 1: Ohne Glättung
//ScaleXY(currentFaceFrame.Rotation, out x, out y);
//MouseControl.Move(x, y);
////Method 2: Glättung über die letzten x Bilder:
//int i = 0;
//Vector3DF rotationMedium = new Vector3DF();
//while (i < 10 && headRotationHistory.Count - 1 > i)
//{
// i++;
// rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X;
// rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y;
//}
//rotationMedium.X = rotationMedium.X / i;
//rotationMedium.Y = rotationMedium.Y / i;
//ScaleXY(rotationMedium, out x, out y);
//MouseControl.Move(x, y);
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < gaussFilter.Count - 1)
{
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
i++;
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
ScaleXY(rotationMedium, out x, out y);
//Method 4: Quadratische Glättung
//double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y));
//double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X));
//if (deltaX < 0)
// deltaX = -Math.Pow(deltaX, 2) * 4;
//else
// deltaX = Math.Pow(deltaX, 2) * 4;
//if (deltaY < 0)
// deltaY = -Math.Pow(deltaY, 2) * 5;
//else
// deltaY = Math.Pow(deltaY, 2) * 5;
//MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY));
//Check for right, left or Double Click
//1. Check if there was already a click 20 Frames ago, or if Drag & Drop is active
if (clickDelay > nudClickDelay.Value && !pointNClickActive)
{
//2. If not, calculate mean values of dy's last 16 Frames
if (CalculateMeanConvolutionValues())
{
clickDelay = 0;
}
else
{
//Else check for open Mouth
if (mouthOpenValue > (float)nudMouthOpenStartThreshold.Value && mouthOpenHistory[mouthOpenHistory.Count - 2] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 3] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 4] > (float)nudMouthOpenConfirmation.Value)
{
MouseControl.Move(mousePositionHistory[mousePositionHistory.Count - 4].X, mousePositionHistory[mousePositionHistory.Count - 4].Y);
this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y)));
//lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y);
MouseControl.MouseDownLeft();
pointNClickActive = true;
clickDelay = 0;
}
}
}
else if (pointNClickActive)
{
if (mouthOpenValue < (float)nudMouthOpenEndThreshold.Value)
{
this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Up on X: " + x + " Y: " + y)));
MouseControl.MouseUpLeft();
pointNClickActive = false;
clickDelay = 0;
}
}
MouseControl.Move(x, y);
if (browLowererValue > (float)nudBrowLowererStartThreshold.Value)
{
MouseControl.ScrollDown((int)(-browLowererValue * (int)nudScrollMultiplierDown.Value));
}
if (browRaiserValue > (float)nudBrowRaiserStartThreshold.Value)
{
MouseControl.ScrollDown((int)(browRaiserValue * (int)nudScrollMultiplierUp.Value));
}
if (mousePositionHistory.Count > 100)
{
mousePositionHistory.RemoveAt(0);
}
mousePositionHistory.Add(new Microsoft.Kinect.Toolkit.FaceTracking.Point(x, y));
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face and eyes successfully tracked.\n");
}
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face recognized but couldn't find eye in face.\n");
}
clickDelay++;
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Active Skeleton is null. Couldn't analyze frame.\n");
}
}
catch (Exception e)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Error during frame analyzation.\n" + e.ToString());
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
