private bool _disposedValue = false; // To detect redundant calls
protected virtual void Dispose(bool disposing)
{
if (!_disposedValue)
{
if (disposing)
{
_skeletonFrame?.Dispose();
}
_disposedValue = true;
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData); // An array of skeletons
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
// Pass the skeleton to the skeletonFaceTracker - see that class below
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
this.rotation = skeletonFaceTracker.rotation;
// Draw the skeleton here...
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private void AllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for changes in any of the data this function is receiving
// and reset things appropriately.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.DestroyFaceTracker();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.DestroyFaceTracker();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
this.colorImageWritableBitmap = null;
this.ColorImage.Source = null;
this.theMaterial.Brush = null;
}
if (this.skeletonData != null && this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = null;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
if (this.colorImageWritableBitmap == null)
{
this.colorImageWritableBitmap = new WriteableBitmap(
colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null);
this.ColorImage.Source = this.colorImageWritableBitmap;
this.theMaterial.Brush = new ImageBrush(this.colorImageWritableBitmap)
{
ViewportUnits = BrushMappingMode.Absolute
};
}
if (this.skeletonData == null)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
// Copy data received in this event to our buffers.
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
this.colorImageWritableBitmap.WritePixels(
new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height),
this.colorImage,
colorImageFrame.Width * Bgr32BytesPerPixel,
0);
// Find a skeleton to track.
// First see if our old one is good.
// When a skeleton is in PositionOnly tracking state, don't pick a new one
// as it may become fully tracked again.
Skeleton skeletonOfInterest =
this.skeletonData.FirstOrDefault(
skeleton =>
skeleton.TrackingId == this.trackingId &&
skeleton.TrackingState != SkeletonTrackingState.NotTracked);
if (skeletonOfInterest == null)
{
// Old one wasn't around. Find any skeleton that is being tracked and use it.
skeletonOfInterest =
this.skeletonData.FirstOrDefault(
skeleton => skeleton.TrackingState == SkeletonTrackingState.Tracked);
if (skeletonOfInterest != null)
{
// This may be a different person so reset the tracker which
// could have tuned itself to the previous person.
if (this.faceTracker != null)
{
this.faceTracker.ResetTracking();
}
this.trackingId = skeletonOfInterest.TrackingId;
}
}
bool displayFaceMesh = false;
if (skeletonOfInterest != null && skeletonOfInterest.TrackingState == SkeletonTrackingState.Tracked)
{
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(this.Kinect);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame faceTrackFrame = this.faceTracker.Track(
this.colorImageFormat,
this.colorImage,
this.depthImageFormat,
this.depthImage,
skeletonOfInterest);
if (faceTrackFrame.TrackSuccessful)
{
this.UpdateMesh(faceTrackFrame);
// Only display the face mesh if there was a successful track.
displayFaceMesh = true;
}
}
}
else
{
this.trackingId = -1;
}
this.viewport3d.Visibility = displayFaceMesh ? Visibility.Visible : Visibility.Hidden;
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
static void SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
{
if ((object)vid == null)
{
return;
}
while (imageSyncBlocked)
{
;
}
imageSyncBlocked = true;
b = PImageToBitmap(vid);
imageSyncBlocked = false;
SkeletonFrame skellyIn = e.OpenSkeletonFrame();
Skeleton[] skellies = new Skeleton[skellyIn.SkeletonArrayLength];
skellyIn.CopySkeletonDataTo(skellies);
//if(skellies.Skeletons[0].TrackingState.ToString()!="NotTracked")
//Console.WriteLine(skellies.Skeletons[0].TrackingState.ToString());
JointCollection j;
bool isReady = false;
if (skellies[currSkelly].TrackingState == SkeletonTrackingState.PositionOnly || skellies[currSkelly].TrackingState == SkeletonTrackingState.Tracked)
{
isReady = true;
}
else
{
f.setTrackingL("Seeking target...");
for (int i = 0; i < 6; i++)
{
currSkelly++;
currSkelly = i;
if (skellies[currSkelly].TrackingState == SkeletonTrackingState.PositionOnly || skellies[currSkelly].TrackingState == SkeletonTrackingState.Tracked)
{
isReady = true;
f.setTrackingL("Target Acquired");
break;
}
}
}
if (isReady)
{
j = skellies[currSkelly].Joints;
Joint h;
float xd, yd;
h = j[JointType.ElbowRight];
findJoint(h.Position, out xd, out yd);
drawPoint((int)xd, (int)yd);
//elbow.recPoint(h.Position);
h = j[JointType.HandRight];
findJoint(h.Position, out xd, out yd);
drawPoint((int)xd, (int)yd);
hand.recPoint(h.Position);
SkeletonPoint temp = hand.averageVel(50, .6f);
xd = temp.X;
yd = temp.Y;
sendData();
}
draw(f);
skellyIn.Dispose();
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
// Update the list of trackers and the trackers with the current frame information
var i = 0;
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
var position = skeleton.Position;
if (position.X != 0 && position.Y != 0 && position.Z != 0)
{
Console.WriteLine($"Face {i}: X {position.X}, Y {position.Y}, Z {position.Z}");
string text = position.X + "\r\n" + position.Y + "\r\n" + position.Z;
System.IO.File.WriteAllText(@"C:\Users\Mykayla\Desktop\transforms\coords_" + file_index + ".txt", text);
file_index++;
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
i += 1;
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
void KinectFaceNode_AllFrameReady(object sender, AllFramesReadyEventArgs e)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
colorImageFrame = e.OpenColorImageFrame();
depthImageFrame = e.OpenDepthImageFrame();
skeletonFrame = e.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
return;
}
if (first)
{
first = false;
this.olddepth = depthImageFrame.Format;
}
else
{
if (this.olddepth != depthImageFrame.Format)
{
//Need a reset
if (this.depthImage != null)
{
this.depthImage = null;
}
foreach (SkeletonFaceTracker sft in this.trackedSkeletons.Values)
{
sft.Dispose();
}
this.trackedSkeletons.Clear();
this.olddepth = depthImageFrame.Format;
}
}
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.runtime.Runtime, colorImageFrame.Format, colorImage, depthImageFrame.Format, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
colorImageFrame.Dispose();
depthImageFrame.Dispose();
skeletonFrame.Dispose();
this.FInvalidate = true;
}
public void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
void KinectFaceNode_AllFrameReady(object sender, AllFramesReadyEventArgs e)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
if (face == null)
{
face = new FaceTracker(this.runtime.Runtime);
}
colorImageFrame = e.OpenColorImageFrame();
depthImageFrame = e.OpenDepthImageFrame();
skeletonFrame = e.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.runtime.Runtime, ColorImageFormat.RgbResolution640x480Fps30, colorImage, DepthImageFormat.Resolution320x240Fps30, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
colorImageFrame.Dispose();
depthImageFrame.Dispose();
skeletonFrame.Dispose();
this.FInvalidate = true;
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.resetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.resetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
try
{
this.trackedSkeleton.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
}
catch (NullReferenceException)
{
//se si perder il tracking del viso si evita un crash
}
this.trackedSkeleton.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
/// <summary>
/// Event called when the kinect believes new skeletal data exits (Player moved).
/// Event then collects data on wanted joints
/// </summary>
private void kinectSensor_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
{
//If the player is using the kinect controller
if (isKinectController)
{
//Retrieve players
SkeletonFrame frame = e.OpenSkeletonFrame();
if (frame == null)
{
return;
}
//Store joints from players
Skeleton[] players = new Skeleton[frame.SkeletonArrayLength];
frame.CopySkeletonDataTo(players);
//Grab the first player that is being tracked
Skeleton firstPlayer = null;
for (int i = 0; i < players.Length; ++i)
{
if (players[i].TrackingState == SkeletonTrackingState.Tracked || players[i].TrackingState == SkeletonTrackingState.PositionOnly)
{
firstPlayer = players[i];
}
}
//Dispose the non tracking frame
if (firstPlayer == null)
{
Player.gameState = Player.States.noplayer;
frame.Dispose();
return;
}
//If there was no player found in the game
if (Player.gameState == Player.States.noplayer)
{
//Change the state to setup
Player.gameState = Player.States.setup;
}
//Store the left and right joints for later scaling
leftHand = firstPlayer.Joints[JointType.HandLeft];
rightHand = firstPlayer.Joints[JointType.HandRight];
//Remove previous points from Dictionary
skeletonPoints.Clear();
//Map specific joints
int joint = 0;
for (int i = 0; i < 2; ++i)
{
//Update the left arm joint
if (i == 0)
{
joint = 7;
}
//Update the right arm joint
else
{
joint = 11;
}
//Retrienve the joint type
JointType type = ((JointType[])Enum.GetValues(typeof(JointType)))[joint];
//Map the joint point on the ColorImage
ColorImagePoint mappedPoints;
mappedPoints = kinectSensor.CoordinateMapper.MapSkeletonPointToColorPoint(firstPlayer.Joints[type].Position,
ColorImageFormat.RgbResolution640x480Fps30);
//Add mapped Coordinate into dictionary
skeletonPoints.Add(type, mappedPoints);
}
frame.Dispose();
}
}
private void OnAllFramesReady(object sender, Microsoft.Kinect.AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.SkeletonData);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
//Idea: Separate Eye-Parts of Color Image
//Use learning Algorithm for right and left eye
//Detect blink on separated parts of color Image
//colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values
if (activeSkeleton != null)
{
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser];
float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower];
tbBrowLowerer.Text = browLowererValue.ToString();
tbBrowRaiser.Text = browRaiserValue.ToString();
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
pbLeftEye.Image = leftEye;
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
pbRightEye.Image = rightEye;
//Wende Kantenfilter auf die beiden Augen an.
double dxRight;
double dyRight;
double dxLeft;
double dyLeft;
if (rightEye != null && leftEye != null)
{
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out dxRight, out dyRight);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out dxLeft, out dyLeft);
//If Face is rotated, move Mouse
if (headRotationHistory.Count > filterLength && currentFaceFrame.TrackSuccessful)
{
int x = 0;
int y = 0;
//Method 1: Ohne Glättung
//ScaleXY(currentFaceFrame.Rotation, out x, out y);
//MouseControl.Move(x, y);
////Method 2: Glättung über die letzten x Bilder:
//int i = 0;
//Vector3DF rotationMedium = new Vector3DF();
//while (i < 10 && headRotationHistory.Count - 1 > i)
//{
// i++;
// rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X;
// rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y;
//}
//rotationMedium.X = rotationMedium.X / i;
//rotationMedium.Y = rotationMedium.Y / i;
//ScaleXY(rotationMedium, out x, out y);
//MouseControl.Move(x, y);
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < filterLength - 1)
{
i++;
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
ScaleXY(rotationMedium, out x, out y);
MouseControl.Move(x, y);
//Method 4: Quadratische Glättung
//double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y));
//double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X));
//if (deltaX < 0)
// deltaX = -Math.Pow(deltaX, 2) * 4;
//else
// deltaX = Math.Pow(deltaX, 2) * 4;
//if (deltaY < 0)
// deltaY = -Math.Pow(deltaY, 2) * 5;
//else
// deltaY = Math.Pow(deltaY, 2) * 5;
//MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY));
}
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
}
}
catch (Exception e)
{
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private static void Nui_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
{
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) // Open the Skeleton frame
{
if (skeletonFrame != null) // check that a frame is available
{
Skeleton[] skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
int TotalSkeletons = skeletonFrame.SkeletonArrayLength;
int counter = 0;
skeletonFrame.CopySkeletonDataTo(skeletons); // get the skeletal information in this frame
foreach (Skeleton S in skeletons)
{
if (S.ClippedEdges.HasFlag(FrameEdges.Left) || S.ClippedEdges.HasFlag(FrameEdges.Right))
{
S.TrackingState = SkeletonTrackingState.NotTracked;
closestdistance = 10000f;
closestid = 0;
skeletonFrame.Dispose();
}
if (S.TrackingState != SkeletonTrackingState.NotTracked)
{
if (S.Position.Z < closestdistance)
{
closestid = S.TrackingId;
closestdistance = S.Position.Z;
}
if (closestid > 0)
{
sensor.SkeletonStream.ChooseSkeletons(closestid); // track this skeleton
if (S.TrackingId == closestid)
{
string json = S.Serialize();
foreach (var socket in _sockets)
{
socket.Send(json); // send JSON via websocket
}
}
}
}
if (S.TrackingState == SkeletonTrackingState.NotTracked)
{
counter++;
if (counter == TotalSkeletons)
{
closestdistance = 10000f;
closestid = 0;
}
}
}
}
else
{
// beware of using loops!
}
}
}
private void OnSkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs eventArgs)
{
SkeletonFrame skeletonFrame = null;
try
{
skeletonFrame = eventArgs.OpenSkeletonFrame();
if (skeletonFrame == null)
{
return;
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
int min1id = 0;
int min2id = 0;
float min1dist = maxDistance;
float min2dist = maxDistance;
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState != SkeletonTrackingState.NotTracked)
{
if (skeleton.Position.Z > minDistance)
{
if (min2dist <= min1dist && skeleton.Position.Z < min1dist)
{
min1dist = skeleton.Position.Z;
min1id = skeleton.TrackingId;
}
else if (min1dist <= min2dist && skeleton.Position.Z < min2dist)
{
min2dist = skeleton.Position.Z;
min2id = skeleton.TrackingId;
}
}
}
}
if (min1id > 0 && min2id > 0)
{
kinectSensor.SkeletonStream.ChooseSkeletons(min1id, min2id);
}
else if (min1id > 0)
{
kinectSensor.SkeletonStream.ChooseSkeletons(min1id);
}
else
{
kinectSensor.SkeletonStream.ChooseSkeletons();
}
KinectBodySet bodies = new KinectBodySet();
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked && skeleton.Joints[JointType.Head].TrackingState == JointTrackingState.Tracked)
{
KinectBody body = new KinectBody();
body.Id = (ulong)skeleton.TrackingId;
SkeletonPoint head = skeleton.Joints[JointType.Head].Position;
body.Head = new KinectBodyPart(head.X, head.Y, head.Z);
if (skeleton.Joints[JointType.HandLeft].TrackingState == JointTrackingState.Tracked)
{
SkeletonPoint hand = skeleton.Joints[JointType.HandLeft].Position;
body.HandLeft = new KinectBodyPart(hand.X, hand.Y, hand.Z);
}
if (skeleton.Joints[JointType.HandRight].TrackingState == JointTrackingState.Tracked)
{
SkeletonPoint hand = skeleton.Joints[JointType.HandRight].Position;
body.HandRight = new KinectBodyPart(hand.X, hand.Y, hand.Z);
}
bodies.add(body);
}
}
if (bodies.size() > 0)
{
foreach (BodyListener listener in new List <BodyListener>(bodyListeners))
{
listener.onBodiesReceived(bodies.ToString());
}
}
}
finally
{
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
Trace.WriteLine("allframesready");
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
Trace.WriteLine("try start");
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
Trace.WriteLine("return from frameready");
return;
}
Trace.WriteLine("wayaround null");
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
Trace.WriteLine("depth image fromat no work");
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
Trace.WriteLine("color image fromat no work");
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
Trace.WriteLine("starting copying of data");
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
Trace.WriteLine("end copying of data");
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
Trace.WriteLine("inside foreach loop");
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
Trace.WriteLine("iftracked");
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
Trace.WriteLine("add skeleton tracking");
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
Trace.WriteLine("before");
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton, this.activityWindow);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
File.AppendAllText("mouseLog.txt", DateTime.Now + " - All Kinect frames ready.\n");
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Color- depth or Skeletonframe is null. Aborting Frame.\n");
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.SkeletonData);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
//Idea: Separate Eye-Parts of Color Image
//Use learning Algorithm for right and left eye
//Detect blink on separated parts of color Image
//colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values
if (activeSkeleton != null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Skeleton is there. Trying to find face.\n");
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
if (currentFaceFrame.TrackSuccessful)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Recognized face successfully.\n");
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Couldn't find face in frame.\n");
}
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
//this.pbRight.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye));
//
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
Bitmap leftEye2 = null;
Bitmap rightEye2 = null;
if (leftEye != null)
{
leftEye2 = new Bitmap(leftEye);
}
if (rightEye != null)
{
rightEye2 = new Bitmap(rightEye);
}
// System.Delegate d = new MethodInvoker(SetPictures));
// this.Invoke(SetPictures, leftEye);
//pbRight.Image = rightEye;
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbLeft.Image = rightEye2));
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye2));
// this.Invoke(new MethodInvoker(SetPictures));
//Wende Kantenfilter auf die beiden Augen an.
if (rightEye != null && leftEye != null)
{
Dictionary <string, int> angleCount;
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out angleCount);
bool rightEyeClosed = IsEyeClosed(angleCount);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out angleCount);
bool leftEyeClosed = IsEyeClosed(angleCount);
// pbLeftFaltung.Image = edgePicLeft;
// pbRightFaltung.Image = edgePicRight;
if (rightEyeClosedHistory.Count > 100)
{
rightEyeClosedHistory.RemoveAt(0);
}
if (leftEyeClosedHistory.Count > 100)
{
leftEyeClosedHistory.RemoveAt(0);
}
leftEyeClosedHistory.Add(leftEyeClosed);
rightEyeClosedHistory.Add(rightEyeClosed);
//If Face is rotated, move Mouse
if (headRotationHistory.Count > gaussFilter.Count - 1 && leftEyeClosedHistory.Count > nudConvolutionFilterLength.Value && currentFaceFrame.TrackSuccessful)
{
int x = 0;
int y = 0;
float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser];
float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower];
float mouthOpenValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.JawLower];
if (browRaiserHistory.Count >= 100)
{
browRaiserHistory.RemoveAt(0);
browLowererHistory.RemoveAt(0);
mouthOpenHistory.RemoveAt(0);
}
browLowererHistory.Add(browLowererValue);
browRaiserHistory.Add(browRaiserValue);
mouthOpenHistory.Add(mouthOpenValue);
//Method 1: Ohne Glättung
//ScaleXY(currentFaceFrame.Rotation, out x, out y);
//MouseControl.Move(x, y);
////Method 2: Glättung über die letzten x Bilder:
//int i = 0;
//Vector3DF rotationMedium = new Vector3DF();
//while (i < 10 && headRotationHistory.Count - 1 > i)
//{
// i++;
// rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X;
// rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y;
//}
//rotationMedium.X = rotationMedium.X / i;
//rotationMedium.Y = rotationMedium.Y / i;
//ScaleXY(rotationMedium, out x, out y);
//MouseControl.Move(x, y);
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < gaussFilter.Count - 1)
{
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
i++;
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
ScaleXY(rotationMedium, out x, out y);
//Method 4: Quadratische Glättung
//double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y));
//double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X));
//if (deltaX < 0)
// deltaX = -Math.Pow(deltaX, 2) * 4;
//else
// deltaX = Math.Pow(deltaX, 2) * 4;
//if (deltaY < 0)
// deltaY = -Math.Pow(deltaY, 2) * 5;
//else
// deltaY = Math.Pow(deltaY, 2) * 5;
//MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY));
//Check for right, left or Double Click
//1. Check if there was already a click 20 Frames ago, or if Drag & Drop is active
if (clickDelay > nudClickDelay.Value && !pointNClickActive)
{
//2. If not, calculate mean values of dy's last 16 Frames
if (CalculateMeanConvolutionValues())
{
clickDelay = 0;
}
else
{
//Else check for open Mouth
if (mouthOpenValue > (float)nudMouthOpenStartThreshold.Value && mouthOpenHistory[mouthOpenHistory.Count - 2] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 3] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 4] > (float)nudMouthOpenConfirmation.Value)
{
MouseControl.Move(mousePositionHistory[mousePositionHistory.Count - 4].X, mousePositionHistory[mousePositionHistory.Count - 4].Y);
this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y)));
//lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y);
MouseControl.MouseDownLeft();
pointNClickActive = true;
clickDelay = 0;
}
}
}
else if (pointNClickActive)
{
if (mouthOpenValue < (float)nudMouthOpenEndThreshold.Value)
{
this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Up on X: " + x + " Y: " + y)));
MouseControl.MouseUpLeft();
pointNClickActive = false;
clickDelay = 0;
}
}
MouseControl.Move(x, y);
if (browLowererValue > (float)nudBrowLowererStartThreshold.Value)
{
MouseControl.ScrollDown((int)(-browLowererValue * (int)nudScrollMultiplierDown.Value));
}
if (browRaiserValue > (float)nudBrowRaiserStartThreshold.Value)
{
MouseControl.ScrollDown((int)(browRaiserValue * (int)nudScrollMultiplierUp.Value));
}
if (mousePositionHistory.Count > 100)
{
mousePositionHistory.RemoveAt(0);
}
mousePositionHistory.Add(new Microsoft.Kinect.Toolkit.FaceTracking.Point(x, y));
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face and eyes successfully tracked.\n");
}
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face recognized but couldn't find eye in face.\n");
}
clickDelay++;
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Active Skeleton is null. Couldn't analyze frame.\n");
}
}
catch (Exception e)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Error during frame analyzation.\n" + e.ToString());
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
