/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException ex)
{
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest).Clone() as FaceTrackFrame;
}
}
// Updates the face tracking information for this skeleton
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
this.facePoints = frame.GetProjected3DShape();
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
if (this.DrawFaceMesh || this.DrawFeaturePoints != DrawFeaturePoint.None)
{
this.facePoints = frame.GetProjected3DShape();
}
// get the shape points array
if (this.DrawShapePoints)
{
// see the !!!README.txt file to add the function
// to your toolkit project
this.shapePoints = frame.GetShapePoints();
}
}
// draw/remove the components
SetFeaturePointsLocations();
SetShapePointsLocations();
}
}
internal Skeleton(SkeletonTrackingState trackingState, int trackingID, int enrollmentIndex, SkeletonPoint position, JointCollection joints)
{
TrackingState = trackingState;
TrackingID = trackingID;
EnrollmentIndex = enrollmentIndex;
Position = position;
Joints = joints;
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor,
ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat,
short[] depthImage, Skeleton skeletonOfInterest) // <---------Skeleton data passed here *****************
{
// Here a skeletonOfInterest is available **********************************
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
// Here is where a skeletonOfInterest is available ***
// Call Track(), passing skeletonOfInterest
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
// Assign the facePoints
this.facePoints = frame.GetProjected3DShape();
// Gets the face data but does not draw the face or the skeleton
// This code gets the yaw, pitch and roll Capture it here
this.rotation = frame.Rotation; // <-- frame is a FaceTrackFrame
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this._skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this._skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this._faceTracker == null)
{
try
{
this._faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException e)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Console.WriteLine(Converter.EncodeError("FaceTracker.OnFrameReady - creating a new FaceTracker threw an InvalidOperationException: " + e.Message));
this._faceTracker = null;
}
}
if (this._faceTracker != null)
{
FaceTrackFrame frame = this._faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this._lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this._lastFaceTrackSucceeded)
{
var animUnits = frame.GetAnimationUnitCoefficients();
var pos = frame.Translation;
var rot = frame.Rotation;
var faceData = new FaceData();
faceData.Au0 = animUnits[0];
faceData.Au1 = animUnits[1];
faceData.Au2 = animUnits[2];
faceData.Au3 = animUnits[3];
faceData.Au4 = animUnits[4];
faceData.Au5 = animUnits[5];
faceData.PosX = pos.X;
faceData.PosY = pos.Y;
faceData.PosZ = pos.Z;
faceData.RotX = rot.X;
faceData.RotY = rot.Y;
faceData.RotZ = rot.Z;
var data = Converter.EncodeFaceTrackingData(faceData);
Console.WriteLine(data);
}
}
}
//private Vector3D transformAndConvert(SkeletonPoint position, Matrix3D rotation)
//{
// Vector3D adjustedVector = new Vector3D(position.X, position.Y, position.Z);
// adjustedVector = Vector3D.Multiply(adjustedVector, rotation);
// return adjustedVector;
//}
//private SkeletonPoint transform(SkeletonPoint position, Matrix3D rotation)
//{
// Point3D adjustedVector = new Point3D(position.X, position.Y, position.Z);
// adjustedVector = Point3D.Multiply(adjustedVector, rotation);
// SkeletonPoint adjustedPoint = new SkeletonPoint();
// adjustedPoint.X = (float)adjustedVector.X;
// adjustedPoint.Y = (float)adjustedVector.Y;
// adjustedPoint.Z = (float)adjustedVector.Z;
// return adjustedPoint;
//}
#endregion
//Misc Methods
private KinectBase.TrackingState convertTrackingState(SkeletonTrackingState trackingState)
{
if (trackingState == SkeletonTrackingState.PositionOnly)
{
//The position only state is out of order, so we have to set it manually
return(KinectBase.TrackingState.PositionOnly);
}
else
{
//All the rest are numbered the same, so we can do a direct cast
return((KinectBase.TrackingState)trackingState);
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
this.test = frame.Get3DShape();
//info about rotations
pitchVal = frame.Rotation.X;
rollVal = frame.Rotation.Z;
yawVal = frame.Rotation.Y;
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
double dbX = facePoints[5].X;
double dbY = facePoints[5].Y;
App thisApp = App.Current as App;
thisApp.m_dbX = dbX;
thisApp.m_dbY = dbY;
}
}
}
public void Set(Skeleton skeleton)
{
State = skeleton.TrackingState;
foreach (Joint joint in skeleton.Joints)
{
var pos = new Vector3(joint.Position.X, -joint.Position.Y, joint.Position.Z);
if (Joints.ContainsKey(joint.JointType))
{
Joints[joint.JointType] = pos;
}
else
{
Joints.Add(joint.JointType, pos);
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest).Clone() as FaceTrackFrame;
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
/*if (faceTriangles == null)
* {
* // only need to get this once. It doesn't change.
* faceTriangles = frame.GetTriangles();
* }
*
* this.facePoints = frame.GetProjected3DShape();*/
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
Vector3DF faceRotation = frame.Rotation;
var AUCoeff = frame.GetAnimationUnitCoefficients();
var jawLower = AUCoeff[AnimationUnit.JawLower];
var BrowLower = AUCoeff[AnimationUnit.BrowLower];
var BrowUpper = AUCoeff[AnimationUnit.BrowRaiser];
var lcd = AUCoeff[AnimationUnit.LipCornerDepressor];
var lipRaiser = AUCoeff[AnimationUnit.LipRaiser];
var lipStrectch = AUCoeff[AnimationUnit.LipStretcher];
var Pitch = faceRotation.X;
var Yaw = faceRotation.Y;
var Roll = faceRotation.Z;
dataToBeSent1 = "P: " + ((float)Pitch).ToString() + " Y: " + ((float)Yaw).ToString() + " R: " + ((float)Roll).ToString();
dataToBeSent2 = "JL: " + ((float)jawLower).ToString() + " BL: " + ((float)BrowLower).ToString() + " BU: " + ((float)BrowUpper).ToString();
dataToBeSent3 = "lcd: " + ((float)lcd).ToString() + " LR: " + ((float)lipRaiser).ToString() + " LS: " + ((float)lipStrectch).ToString();
}
}
}
private bool CheckFace(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return(false);
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
//getting the Animation Unit Coefficients
this.AUs = frame.GetAnimationUnitCoefficients();
var jawLowerer = AUs[AnimationUnit.JawLower];
var browLower = AUs[AnimationUnit.BrowLower];
var browRaiser = AUs[AnimationUnit.BrowRaiser];
var lipDepressor = AUs[AnimationUnit.LipCornerDepressor];
var lipRaiser = AUs[AnimationUnit.LipRaiser];
var lipStretcher = AUs[AnimationUnit.LipStretcher];
//set up file for output
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt"))
{
file.WriteLine("FaceTrack Data, started recording at " + DateTime.Now.ToString("HH:mm:ss tt"));
}
//here is the algorithm to test different facial features
//BrowLower is messed up if you wear glasses, works if you don't wear 'em
string state = "";
//surprised
if ((jawLowerer < 0.25 || jawLowerer > 0.25) && browLower < 0)
{
state = "surprised";
}
//smiling
if (lipStretcher > 0.4 || lipDepressor < 0)
{
state = "smiling";
}
//sad
if (browRaiser < 0 && lipDepressor > 0)
{
state = "sad";
}
//angry
if ((browLower > 0 && (jawLowerer > 0.25 || jawLowerer < -0.25)) ||
(browLower > 0 && lipDepressor > 0))
{
state = "angry";
}
//System.Diagnostics.Debug.WriteLine(browLower);
this.facePoints = frame.GetProjected3DShape();
if (states[currentState] == state)
{
Trace.WriteLine("Yo!");
return(true);
}
}
}
return(false);
}
public UnknownSkeletonTrackingStateException(SkeletonTrackingState s)
: base("Unknown \"SkeletonTrackingState\" value = " + s.ToString())
{
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints3DRaw = frame.Get3DShape();
this.facePoints = frame.GetProjected3DShape();
animationUnitsRaw = frame.GetAnimationUnitCoefficients();
}
x = frame.Rotation.X;
y = frame.Rotation.Y;
z = frame.Rotation.Z;
facePointS3D = this.facePoints3DRaw;
animationUnits = animationUnitsRaw;
//Debug.WriteLine(animationUnits[AnimationUnit.JawLower]);
//Debug.WriteLine(animationUnits[AnimationUnit.BrowLower]);
//Debug.WriteLine(animationUnits[AnimationUnit.BrowRaiser]);
//Debug.WriteLine(animationUnits[AnimationUnit.JawLower]);
//Debug.WriteLine(animationUnits[AnimationUnit.LipCornerDepressor]);
//Debug.WriteLine(animationUnits[AnimationUnit.LipRaiser]);
//Debug.WriteLine(animationUnits[AnimationUnit.LipStretcher]);
//Debug.WriteLine(frame.Translation.ToString());
//Debug.WriteLine(frame.Rotation.ToString());
//this.facePoints[FeaturePoint.AboveChin].X+2;
//Debug.WriteLine(frame.Translation.X.ToString());
//Debug.WriteLine(frame.Translation.Y.ToString());
//Debug.WriteLine(frame.Translation.Z.ToString());
}
}
public void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
XmlDocument XmlDocKalibracja = new XmlDocument();
XmlDocKalibracja.Load(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
XmlNodeList elemListKalibracja = XmlDocKalibracja.GetElementsByTagName("options");
kalibracja = Convert.ToBoolean(Convert.ToInt32(elemListKalibracja[0].Attributes["kalibracja"].Value));
if (kalibracja == false)
{
kalibracjaCounter++;
if (kalibracjaCounter == 1)
{
Kalibracja okno = new Kalibracja();
okno.Show();
}
if (kalibracjaCounter > 150)
{
oldX = Convert.ToInt32(this.facePoints[23].X);
oldY = -Convert.ToInt32(this.facePoints[23].Y);
oldMouseX = System.Windows.Forms.Cursor.Position.X;
oldMouseY = System.Windows.Forms.Cursor.Position.Y;
aktualnyX = oldX;
aktualnyY = oldY;
kalibracja = true;
string newValue = "1";
kalibracjaCounter = 0;
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.Load(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
XmlNode node = xmlDoc.SelectSingleNode("options");
node.Attributes[5].Value = Convert.ToString(newValue);
xmlDoc.Save(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
}
}
if (kalibracja == true)
{
try
{
//ustawienie gestów
XmlDocument XmlDoc = new XmlDocument();
XmlDoc.Load(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
XmlNodeList elemList = XmlDoc.GetElementsByTagName("options");
gest[0] = Convert.ToInt32(elemList[0].Attributes["lpm"].Value);
//ustawienie gestów
XmlDocument XmlDoc2 = new XmlDocument();
XmlDoc2.Load(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
XmlNodeList elemList2 = XmlDoc2.GetElementsByTagName("options");
gest[1] = Convert.ToInt32(elemList2[0].Attributes["ppm"].Value);
//ustawienie gestów
XmlDocument XmlDoc3 = new XmlDocument();
XmlDoc3.Load(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
XmlNodeList elemList3 = XmlDoc3.GetElementsByTagName("options");
gest[2] = Convert.ToInt32(elemList3[0].Attributes["scrollup"].Value);
//ustawienie gestów
XmlDocument XmlDoc4 = new XmlDocument();
XmlDoc4.Load(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
XmlNodeList elemList4 = XmlDoc4.GetElementsByTagName("options");
gest[3] = Convert.ToInt32(elemList4[0].Attributes["scrolldown"].Value);
}
catch
{
MessageBox.Show("Błąd przy odczycie pliku settings.xml");
}
newX = Convert.ToInt32(this.facePoints[23].X);
newY = -Convert.ToInt32(this.facePoints[23].Y);
stosunekX = Math.Abs(newX / oldX);
stosunekY = Math.Abs(newY / oldY);
//odczyt czułości z pliku settings.xml
try
{
XmlDocument XmlDoc = new XmlDocument();
XmlDoc.Load(System.AppDomain.CurrentDomain.BaseDirectory + "\\settings.xml");
XmlNodeList elemList = XmlDoc.GetElementsByTagName("options");
sensitive = Convert.ToInt32(elemList[0].Attributes["sensitive"].Value);
}
catch
{
sensitive = 80;
}
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) < 28 && Math.Abs(oldX - newX) > 70 && Math.Abs(oldY - newY) > 70 && Convert.ToInt32(Math.Abs(oldMouseX + (sensitive * stosunekX - sensitive))) < 1980 || Convert.ToInt32(Math.Abs(oldMouseY + (sensitive * stosunekY - sensitive))) < 1200 && Convert.ToInt32(Math.Abs(oldMouseX + (sensitive * stosunekX))) >= 0 && Convert.ToInt32(Math.Abs(oldMouseY + (sensitive * stosunekY))) >= 0)
{
if (stosunekX > 1.03 && ruchY == false)
{
System.Windows.Forms.Cursor.Position = new System.Drawing.Point(Convert.ToInt32(Math.Abs(oldMouseX + (sensitive * stosunekX - sensitive))), Convert.ToInt32(Math.Abs(oldMouseY)));
oldMouseX = Convert.ToInt32(Math.Abs(oldMouseX + (sensitive * stosunekX - sensitive)));
ruchX = true;
}
if (stosunekX < 0.97 && ruchY == false)
{
System.Windows.Forms.Cursor.Position = new System.Drawing.Point(Convert.ToInt32(Math.Abs(oldMouseX - (sensitive - (sensitive * stosunekX)))), Convert.ToInt32(Math.Abs(oldMouseY)));
oldMouseX = Convert.ToInt32(Math.Abs(oldMouseX - (sensitive - (sensitive * stosunekX))));
ruchX = true;
}
if (stosunekY > 1.03 && ruchX == false)
{
System.Windows.Forms.Cursor.Position = new System.Drawing.Point(Convert.ToInt32(Math.Abs(oldMouseX)), Convert.ToInt32(Math.Abs(oldMouseY + (sensitive * stosunekY - sensitive))));
oldMouseY = Convert.ToInt32(Math.Abs(oldMouseY + (sensitive * stosunekY - sensitive)));
ruchY = true;
}
if (stosunekY < 0.97 && ruchX == false)
{
System.Windows.Forms.Cursor.Position = new System.Drawing.Point(Convert.ToInt32(Math.Abs(oldMouseX)), Convert.ToInt32(Math.Abs(oldMouseY - (sensitive - (sensitive * stosunekY)))));
oldMouseY = Convert.ToInt32(Math.Abs(oldMouseY - (sensitive - (sensitive * stosunekY))));
ruchY = true;
}
}
//stan spoczynku
if (Math.Abs(oldX - newX) < 70)
{
ruchX = false;
}
//stan spoczynku
if (Math.Abs(oldY - newY) < 70)
{
ruchY = false;
}
//PIERWSZY GEST
if (gest[0] == 0)
{
if (this.facePoints[40].Y - this.facePoints[87].Y > 12)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[0] == 1)
{
if (this.facePoints[57].Y - this.facePoints[51].Y > 15)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[0] == 2)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY > 1.03)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[0] == 3)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY < 0.97)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
//DRUGI GEST
if (gest[1] == 0)
{
if (this.facePoints[40].Y - this.facePoints[87].Y > 12)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[1] == 1)
{
if (this.facePoints[57].Y - this.facePoints[51].Y > 14)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[1] == 2)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY > 1.03)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[1] == 3)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY < 0.97)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
//TRZECI GEST
if (gest[2] == 0)
{
if (this.facePoints[40].Y - this.facePoints[87].Y > 12)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[2] == 1)
{
if (this.facePoints[57].Y - this.facePoints[51].Y > 14)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[2] == 2)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY > 1.03)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[2] == 3)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY < 0.97)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
//CZWARTY GEST
if (gest[3] == 0)
{
if (this.facePoints[40].Y - this.facePoints[87].Y > 12)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[3] == 1)
{
if (this.facePoints[57].Y - this.facePoints[51].Y > 14)
{
clickCounter++;
if (clickCounter % 10 == 0)
{
DoMouseClick(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[3] == 2)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY > 1.03)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(1);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
if (gest[3] == 3)
{
if (Math.Abs(this.facePoints[88].X - this.facePoints[89].X) > 28 && stosunekY < 0.97)
{
clickCounter++;
if (clickCounter % 2 == 0)
{
DoMouseScroll(0);
}
if (clickCounter == 10000)
{
clickCounter = 0;
}
}
}
}
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// if the current skeleton is not tracked, track it now
//kinectSensor.SkeletonStream.ChooseSkeletons(skeletonOfInterest.TrackingId);
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
// hack to make this face tracking detect the face even when it is not actually tracked
// <!>need to confirm if it works
//skeletonOfInterest.TrackingState = SkeletonTrackingState.Tracked;
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
//new Microsoft.Kinect.Toolkit.FaceTracking.Rect(skeletonOfInterest.Position.));
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
if (faceTag == null)
{
// here call the face detection
faceTag = new FaceRecognizer().getFaceTag(this.colorImageBmp);
if (faceTag != null)
{
Global.StatusBarText.Text = "Found " + faceTag + "!";
if (Global.trackedPeople.ContainsKey(skeletonOfInterest))
{
Global.trackedPeople[skeletonOfInterest] = faceTag;
}
else
{
Global.trackedPeople.Add(skeletonOfInterest, faceTag);
}
}
}
this.facePoints = frame.GetProjected3DShape();
this.faceRect = frame.FaceRect;
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
//public static System.IO.Ports.SerialPort serialPort1;
//private System.IO.Ports.SerialPort serialPort1;
//serialPort1 = new SerialPort();
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
/*System.ComponentModel.IContainer components = new System.ComponentModel.Container();
serialPort1 = new System.IO.Ports.SerialPort(components); // Creating the new object.
serialPort1.PortName = "COM3"; //+ numCom.Value.ToString(); // Setting what port number.
serialPort1.BaudRate = 9600; // Setting baudrate.
serialPort1.DtrEnable = true; // Enable the Data Terminal Ready
serialPort1.Open(); // Open the port for use.*/
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
rotation = frame.Rotation;
float yRotation = frame.Rotation.Y;
Debug.WriteLine(yRotation);
if ((yRotation <= -25))
{
Debug.WriteLine("a");
//port.Write("a");
if (!serialPort1.IsOpen)
{
try
{
serialPort1.Open();
serialPort1.Write("a");
serialPort1.Close();
}
catch
{
MessageBox.Show("There was an error. Please make sure that the correct port was selected, and the device, plugged in.");
}
}
//serialPort1.Write("1");
}
if ((yRotation > -25) && (yRotation < -10))
{
Debug.WriteLine("b");
//port.Write("a");
if (!serialPort1.IsOpen)
{
try
{
serialPort1.Open();
serialPort1.Write("b");
serialPort1.Close();
}
catch
{
MessageBox.Show("There was an error. Please make sure that the correct port was selected, and the device, plugged in.");
}
}
//serialPort1.Write("1");
}
if ((yRotation >= -10) && (yRotation < 10))
{
Debug.WriteLine("c");
//port.Write("a");
if (!serialPort1.IsOpen)
{
try
{
serialPort1.Open();
serialPort1.Write("c");
serialPort1.Close();
}
catch
{
MessageBox.Show("There was an error. Please make sure that the correct port was selected, and the device, plugged in.");
}
}
//serialPort1.Write("1");
}
if ((yRotation >= 10) && (yRotation < 20))
{
Debug.WriteLine("d");
//port.Write("a");
if (!serialPort1.IsOpen)
{
try
{
serialPort1.Open();
serialPort1.Write("d");
serialPort1.Close();
}
catch
{
MessageBox.Show("There was an error. Please make sure that the correct port was selected, and the device, plugged in.");
}
}
//serialPort1.Write("1");
}
if ((yRotation >= 20) && (yRotation < 30))
{
Debug.WriteLine("e");
//port.Write("a");
if (!serialPort1.IsOpen)
{
try
{
serialPort1.Open();
serialPort1.Write("e");
serialPort1.Close();
}
catch
{
MessageBox.Show("There was an error. Please make sure that the correct port was selected, and the device, plugged in.");
}
}
//serialPort1.Write("1");
}
if ((yRotation >= 30))
{
Debug.WriteLine("f");
//port.Write("a");
if (!serialPort1.IsOpen)
{
try
{
serialPort1.Open();
serialPort1.Write("f");
serialPort1.Close();
}
catch
{
MessageBox.Show("There was an error. Please make sure that the correct port was selected, and the device, plugged in.");
}
}
//serialPort1.Write("1");
}
}
}
}
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
if (!timerStarted)
{
timer.Start();
timerStarted = true;
}
//increment our frames
numberOfFrames++;
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
/*if ()
* {
* Debug.WriteLine("hit " + (frameIter * sampleRate) + " frames in " + (timer.Elapsed) + " seconds");
* frameIter++;
* }*/
//Also grab our points
EnumIndexableCollection <FeaturePoint, Vector3DF> facePoints3D = frame.Get3DShape();
int index = 0;
if (numberOfFrames > frameIter * sampleRate && frameIter < 5) //only grab 4 samples over our given sample rate
{
//Create a new thread so we don't make the visual thread throw up all over the place
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
List <Tuple <float, float, float> > myPoints = new List <Tuple <float, float, float> >();
foreach (Vector3DF vector in facePoints3D)
{
//csv.Append(string.Format("( ({1}, {2}, {3}){4}",vector.X, vector.Y, vector.Z, Environment.NewLine));
myPoints.Add(new Tuple <float, float, float>(vector.X, vector.Y, vector.Z));
index++;
}
calculateDistances(myPoints);
frameIter++;
}).Start();
//once = true;
}
if (frameIter == 5)
{
SetStatusText("Generating histograms...");
Console.WriteLine("We are ready to sample");
foreach (float distance in sampleOneDistances)
{
int sampleOneIndex = (int)Math.Floor(64 * distance / sampleOneMaxDistance);
sampleOneHistogram[sampleOneIndex]++;
}
foreach (float distance in sampleTwoDistances)
{
sampleTwoHistogram[(int)Math.Floor(64 * distance / sampleTwoMaxDistance)]++;
}
foreach (float distance in sampleThreeDistances)
{
sampleThreeHistogram[(int)Math.Floor(64 * distance / sampleThreeMaxDistance)]++;
}
foreach (float distance in sampleFourDistances)
{
sampleFourHistogram[(int)Math.Floor(64 * distance / sampleFourMaxDistance)]++;
}
//Go through histogram and divide by distances
//Get
for (int i = 0; i < sampleOneHistogram.Length; i++)
{
sampleOneHistogram[i] = sampleOneHistogram[i] / sampleOneDistances.Count;
}
for (int i = 0; i < sampleTwoHistogram.Length; i++)
{
sampleTwoHistogram[i] = sampleTwoHistogram[i] / sampleTwoDistances.Count;
}
for (int i = 0; i < sampleThreeHistogram.Length; i++)
{
sampleThreeHistogram[i] = sampleThreeHistogram[i] / sampleThreeDistances.Count;
}
for (int i = 0; i < sampleFourHistogram.Length; i++)
{
sampleFourHistogram[i] = sampleFourHistogram[i] / sampleFourDistances.Count;
}
int iter = 0;
foreach (int count in sampleTwoHistogram)//can iterate through any histogram, they're all of size 65
{
Console.WriteLine("Count for hist1/2/3/4[" + iter + "] is " + count + "/" + sampleOneHistogram[iter] + "/" + sampleThreeHistogram[iter] + "/" + sampleFourHistogram[iter]);
iter++;
}
//Write our histograms to a csv file
String[] sampleOneHistString = Array.ConvertAll(sampleOneHistogram, x => x.ToString());
using (System.IO.StreamWriter file = new System.IO.StreamWriter(testFilePath))
{
file.Write(string.Join(",", Enumerable.Range(1, 65).ToArray()) + Environment.NewLine);
file.Write(string.Join(",", sampleOneHistString));
file.Write(Environment.NewLine);
file.Write(string.Join(",", Array.ConvertAll(sampleTwoHistogram, x => x.ToString())));
file.Write(Environment.NewLine);
file.Write(string.Join(",", Array.ConvertAll(sampleThreeHistogram, x => x.ToString())));
file.Write(Environment.NewLine);
file.Write(string.Join(",", Array.ConvertAll(sampleFourHistogram, x => x.ToString())));
}
//pass that data file to jar
String jarPath = "C:\\Users\\Datalab\\Documents\\GitHub\\WekaClassifier\\jar\\wekaClassifier.jar";
System.Diagnostics.Process clientProcess = new Process();
String jarargs = "C:\\Users\\Datalab\\Documents\\GitHub\\WekaClassifier\\data\\training_data.arff C:\\Users\\Datalab\\Documents\\GitHub\\WekaClassifier\\data\\testFormat.dat";
clientProcess.StartInfo.FileName = "java";
clientProcess.StartInfo.Arguments = "-jar " + jarPath + " " + jarargs;
clientProcess.StartInfo.RedirectStandardOutput = true;
clientProcess.StartInfo.UseShellExecute = false;
clientProcess.Start();
String output = clientProcess.StandardOutput.ReadToEnd();
Console.WriteLine(output);
clientProcess.WaitForExit();
int code = clientProcess.ExitCode;
//write to dat file with 4 histograms averaged
frameIter++; //only do this once (will make conditional evaluate to false. Is this clean and clear? Not really? Do I care? Not particularly. At least it's documented.
ftNumPeople++;
SetPeopleText("People tracked : " + ftNumPeople);
SetStatusText("Status: waiting....");
SetPredictionText("Guess: " + output);
}
}
}
}
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
//elabora un frame si e due no
if (this.rightFrameCount != 0)
{
if (this.rightFrameCount == 2)
{
this.rightFrameCount = 0;
}
else //this.rightFrameCount == 1
{
this.rightFrameCount++;
}
return;
}
this.rightFrameCount++;
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
System.Diagnostics.Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
this.facePoints = frame.GetShapePoints();
}
if (this.pause)
{
return;
}
if (this.analizer.IsSetupComplete)
{
outputBox.AppendText(this.analizer.analizeEmotion(frame));
outputBox.AppendText("\r\n");
outputBox.ScrollToEnd();
}
else
{
outputBox.AppendText(this.analizer.setup(frame));
outputBox.ScrollToEnd();
}
}
}
//Misc Methods
private KinectBase.TrackingState convertTrackingState(SkeletonTrackingState trackingState)
{
if (trackingState == SkeletonTrackingState.PositionOnly)
{
//The position only state is out of order, so we have to set it manually
return KinectBase.TrackingState.PositionOnly;
}
else
{
//All the rest are numbered the same, so we can do a direct cast
return (KinectBase.TrackingState)trackingState;
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
initByteBuffer();
}
/*
hurra, dont need 4th vertex
hurra, dont need 35th vertex
hurra, dont need 36th vertex
hurra, dont need 37th vertex
hurra, dont need 38th vertex
hurra, dont need 39th vertex
hurra, dont need 41th vertex
hurra, dont need 42th vertex
hurra, dont need 43th vertex
*/
this.projectedShapePoints = frame.GetProjected3DShape();
this.shapePoints = frame.Get3DShape();
byte[] buffer = WriteFaceDataToBuffer();
//System.Diagnostics.Debug.WriteLine("Should send {0}", buffer.Length);//1468
//need to reduce by 444 bytes
//a point is 12. 85 points would result in 1020 bytes. 4 bytes remaining for face id...?
//currently 121 points. which ones can be leaft out?
//found 9 not needed. still 27 to much...
//header is useless. 434 bytes remaining after removing it
//byte[] buffer = buffer = new byte[1024];//works fast, even if unknown send to ip..
//ushort thefaceid = 22;
//Array.Copy(System.BitConverter.GetBytes(thefaceid), 0, buffer, 0, sizeof(ushort));
try
{
sending_socket.SendTo(buffer, sending_end_point); // is this blocking? need to start sending asynchronously?!?
//System.Diagnostics.Debug.WriteLine("sending {0} bytes to ip {1} on port {2}", buffer.Length, sending_end_point.Address, sending_end_point.Port);
//Console.WriteLine( buffer);
}
catch (Exception send_exception)
{
System.Diagnostics.Debug.WriteLine("Exception {0}", send_exception.Message);
//System.Diagnostics.Debug.WriteLine("Is the buffer with it's {0} bytes to long to send in one packet?", buffer.Length);
}
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
///
public void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
//No Touchy
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
return;
}
if (faceTracker == null)
{
faceTracker = new FaceTracker(kinectSensor);
}
frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
//Touchy
//Assign Reference points
this.absfacePoints = frame.Get3DShape();
leftForehead = this.absfacePoints[FeaturePoint.TopLeftForehead];
rightForehead = this.absfacePoints[FeaturePoint.TopRightForehead];
jaw = this.absfacePoints[FeaturePoint.BottomOfChin];
faceRotationX = frame.Rotation.X;
faceRotationY = frame.Rotation.Y;
faceRotationZ = frame.Rotation.Z;
//Calculate Reference Points
foreheadReferencePointX = ((rightForehead.X - leftForehead.X) / 2);
foreheadReferencePointY = ((rightForehead.Y - leftForehead.Y) / 2);
foreheadReferencePointZ = ((rightForehead.Z - leftForehead.Z) / 2);
//Set Animation Units
AUCoeff = frame.GetAnimationUnitCoefficients();
jawLowererAU = AUCoeff[AnimationUnit.JawLower];
lipStretcherAU = AUCoeff[AnimationUnit.LipStretcher];
browRaiserAU = AUCoeff[AnimationUnit.BrowRaiser];
setJawData(jaw.Y, leftForehead.Y, rightForehead.Y, jawLowererAU, lipStretcherAU);
rotations = new float[5];
//set up matlab
matlab = new MLApp.MLApp();
matlab.Execute(@"cd C:\Users\Bala\Documents\MATLAB");
result = null;
//get rotation values
rotations[0] = faceRotationX;
rotations[1] = faceRotationY;
rotations[2] = faceRotationZ;
rotations[3] = jawLowererAU;
rotations[4] = lipStretcherAU;
//Set up GlovePie
OscPacket.LittleEndianByteOrder = false;
IPEndPoint myapp = new IPEndPoint(IPAddress.Loopback, 1944);
IPEndPoint glovepie = new IPEndPoint(IPAddress.Loopback, 1945);
Console.WriteLine(browRaiserAU);
matlab.Feval("nnW", 1, out result, rotations[0]);
object[] resW = result as object[];
nnoutput = (int)((float)resW[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 1;
}
else
{
result = null;
matlab.Feval("nnA", 1, out result, rotations[1]);
object[] resA = result as object[];
nnoutput = (int)((float)resA[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 2;
}
else
{
result = null;
matlab.Feval("nnS", 1, out result, rotations[0]);
object[] resS = result as object[];
nnoutput = (int)((float)resS[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 3;
}
else
{
result = null;
matlab.Feval("nnd", 1, out result, rotations[1]);
object[] resD = result as object[];
nnoutput = (int)((float)resD[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 4;
}
else
{
result = null;
matlab.Feval("nnLC", 1, out result, rotations[2]);
object[] resLC = result as object[];
nnoutput = (int)((float)resLC[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 5;
}
else
{
result = null;
matlab.Feval("nnRC", 1, out result, rotations[2]);
object[] resRC = result as object[];
nnoutput = (int)((float)resRC[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 6;
}
else
{
result = null;
if (jawLowererAU > 0.7)
{
commandtoSend = 7;
}
/*
* matlab.Feval("nnSpace", 1, out result, rotations[3]);
* object[] resSpace = result as object[];
* nnoutput = (int)((float)resSpace[0] + 0.5f);
* if (nnoutput == 1)
* {
* commandtoSend = 7;
* }*/
else
{
result = null;
if (browRaiserAU > 0.4)
{
commandtoSend = 8;
}
else
{
result = null;
commandtoSend = 0;
}
/*result = null;
* matlab.Feval("nnMiddle", 1, out result, lipStretcherAU);
* object[] resMiddle = result as object[];
* nnoutput = (int)((float)resMiddle[0] + 0.5f);
* if (nnoutput == 1)
* {
* commandtoSend = 8;
* }
* else
* {
* result = null;
* commandtoSend = 0;
* }*/
}
}
}
}
}
}
}
//Console.WriteLine("Iteration Complete");
switch (commandtoSend)
{
case 0:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 1:
msg = new OscMessage(myapp, "/move/w", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 2:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 3:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 4:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 5:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 6:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 7:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 8:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 10.0f);
msg.Send(glovepie);
break;
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// if the current skeleton is not tracked, track it now
//kinectSensor.SkeletonStream.ChooseSkeletons(skeletonOfInterest.TrackingId);
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
// hack to make this face tracking detect the face even when it is not actually tracked
// <!>need to confirm if it works
//skeletonOfInterest.TrackingState = SkeletonTrackingState.Tracked;
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
//new Microsoft.Kinect.Toolkit.FaceTracking.Rect(skeletonOfInterest.Position.));
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
if (faceTag == null)
{
// here call the face detection
faceTag = new FaceRecognizer().getFaceTag(this.colorImageBmp);
if (faceTag != null)
{
Global.StatusBarText.Text = "Found " + faceTag + "!";
if (Global.trackedPeople.ContainsKey(skeletonOfInterest))
Global.trackedPeople[skeletonOfInterest] = faceTag;
else
Global.trackedPeople.Add(skeletonOfInterest, faceTag);
}
}
this.facePoints = frame.GetProjected3DShape();
this.faceRect = frame.FaceRect;
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked) {
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null) {
try {
this.faceTracker = new FaceTracker(kinectSensor);
} catch (InvalidOperationException) {
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null) {
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded) {
if (faceTriangles == null) {
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
int top = frame.FaceRect.Top;
int left = frame.FaceRect.Left;
int width = frame.FaceRect.Right - frame.FaceRect.Left;
int height = frame.FaceRect.Bottom - frame.FaceRect.Top;
this.faceCoordinates = new int[] { left, top, width, height };
this.rotationXYZ = new float[] { frame.Rotation.X, frame.Rotation.Y, frame.Rotation.Z };
this.translationXYZ = new float[] { frame.Translation.X, frame.Translation.Y, frame.Translation.Z };
} else {
this.rotationXYZ = null;
this.faceCoordinates = null;
this.translationXYZ = null;
}
}
}
private void setSkeleton(Skeleton skel)
{
this.head = skel.Joints[JointType.Head];
this.rightShoulder = skel.Joints[JointType.ShoulderRight];
this.leftShoulder = skel.Joints[JointType.ShoulderLeft];
this.centerShoulder = skel.Joints[JointType.ShoulderCenter];
this.rightFoot = skel.Joints[JointType.FootRight];
this.leftFoot = skel.Joints[JointType.FootLeft];
this.leftHand = skel.Joints[JointType.HandLeft];
this.rightHand = skel.Joints[JointType.HandRight];
this.spine = skel.Joints[JointType.Spine];
this.trackingState = skel.TrackingState;
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
double dbX = facePoints[5].X;
double dbY = facePoints[5].Y;
App thisApp = App.Current as App;
thisApp.m_dbX = dbX;
thisApp.m_dbY = dbY;
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
// int cursorX = 500;
// int cursorY = 500;
// bool click = false;
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
if (Globals.cambio == false)
{
int Pitch = (int)frame.Rotation.X;
int Yaw = (int)frame.Rotation.Y;
/* if ((Yaw > 20) && (Pitch > 0) || (Yaw < -20) && (Pitch > 0) || (Yaw < -20) && (Pitch < -10) || (Yaw > 20) && (Pitch < -10)) //si va en diagonal
* {
* if ((Yaw > 20) && (Pitch > 0))//arribaderecha
* {
* Globals.YAW = 1;
* Globals.PITCH = 1;
* }
* if ((Yaw < -20) && (Pitch > 0))//abajoderecha
* {
* Globals.YAW = -1;
* Globals.PITCH = 1;
* }
* if ((Yaw < -20) && (Pitch < -10))//abajoizda
* {
* Globals.YAW = -1;
* Globals.PITCH = -1;
* }
* if ((Yaw > 20) && (Pitch < -10))//arribaizda
* {
* Globals.YAW = 1;
* Globals.PITCH = -1;
* }
* }
* else
* { */
if (Yaw > 20) //izquierda
{
Globals.YAW = 1;
}
if (Yaw < -20) //derecha
{
Globals.YAW = -1;
}
if ((Yaw < 20) && (Yaw > -20))
{
Globals.YAW = 0;
}
if (Pitch > 0) //arriba
{
Globals.PITCH = 1;
}
if (Pitch < -10) //abajo
{
Globals.PITCH = -1;
}
if ((Pitch < 0) && (Pitch > -10))
{
Globals.PITCH = 0;
}
//}
}
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
isFaceTracked = false;
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
isFaceTracked = true;
this.facePoints = frame.GetProjected3DShape();
this.facePoints3D = frame.Get3DShape(); //****************自己家的
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
List<Point> faceModelPtsCompare = new List<Point>();
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null )
{
double shoulderLeft = (double)skeletonOfInterest.Joints[JointType.ShoulderLeft].Position.X;
double shoulderRight = (double)skeletonOfInterest.Joints[JointType.ShoulderRight].Position.X;
double head = (double)skeletonOfInterest.Joints[JointType.Head].Position.Y;
double hip = (double)skeletonOfInterest.Joints[JointType.HipCenter].Position.Y;
double headDistZ = (double)skeletonOfInterest.Joints[JointType.Head].Position.Z;
shoulderLeft = Math.Abs(shoulderLeft);
shoulderRight = Math.Abs(shoulderRight);
head = Math.Abs(head);
hip = Math.Abs(hip);
headDistZ = Math.Abs(headDistZ);
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.facePoints2 = frame.GetProjected3DShape();
for (int i = 0; i < this.facePoints2.Count; i++)
{
faceModelPtsCompare.Add(new Point(this.facePoints2[i].X + 0.5f, this.facePoints2[i].Y + 0.5f));
}
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (saveFace == true)
{
// trackedSkeletonsSave.Add(skeletonOfInterest.TrackingId, this);
this.facePoints = frame.GetProjected3DShape();
for (int i = 0; i < this.facePoints.Count; i++)
{
faceModelPts.Add(new Point(this.facePoints[i].X + 0.5f, this.facePoints[i].Y + 0.5f));
}
shoulderDistX = (shoulderLeft + shoulderRight) * 100;
headhipDistY = (hip +head) * 100;
saveFaceList.Add(faceModelPts);
saveSkeleList.Add(skeletonOfInterest);
saveTrackList.Add(skeletonOfInterest.TrackingId);
saveFace = false;
}
if (this.lastFaceTrackSucceeded)
{
foreach (Faces kinects in faceList)
{
if (colorCount < 3)
{
if (kinectSensor == kinects.kinect)
{
break;
}
}
else
colorCount = 0;
colorCount++;
}
int countPt = 0;
double pointxDiff = 0;
double pointyDiff = 0;
double pointsNew = 0;
double pointsSaved = 0;
double shoulderDistDiff = 0;
double headhipDistDiff = 0;
double pointsDiff = 0;
if (faceModelPts.Count > 0)
{
foreach (Point pointNew in faceModelPtsCompare)
{
pointsNew = (pointNew.X + pointNew.Y) / headDistZ ;
}
foreach (Point pointSave in faceModelPts)
{
//pointxDiff = (pointNew.X - pointSave.X);
//pointyDiff = (pointNew.Y - pointSave.Y);
pointsSaved = (pointSave.X + pointSave.Y) / headDistZ ;
}
shoulderDistXNew = (shoulderLeft + shoulderRight) ;
headhipDistYNew = (hip + head);
// Debug.WriteLine("x diff: " + pointxDiff + " y diff: " + pointyDiff);
//Debug.WriteLine("new: " + pointsNew + " old: " + pointsSaved);
shoulderDistDiff = Math.Abs(shoulderDistXNew - shoulderDistX) *10 /headDistZ;
headhipDistDiff = Math.Abs(headhipDistYNew - headhipDistY)*10 /headDistZ;
pointsSaved = Math.Abs(pointsNew - pointsSaved) *10;
fMap.newFace = pointsNew;
fMap.oldFace = pointsSaved;
Debug.WriteLine("Shoulder Dist diff: " + shoulderDistDiff / headDistZ);
Debug.WriteLine("Head dist diff: " + headhipDistDiff / headDistZ);
Debug.WriteLine("face points diff: " + pointsSaved / headDistZ);
if (pointsSaved < (50 / headDistZ) )
{
colorCount = 3;
}
}
switch (colorCount)
{
case 0:
rectangle.Width = frame.FaceRect.Width;
rectangle.Height = frame.FaceRect.Height;
Point rectPt = new Point();
rectPt.X = frame.FaceRect.Left;
rectPt.Y = frame.FaceRect.Top;
rectangle.Location = (Point)rectPt;
drawNum = 0;
break;
case 1:
rectangle2.Width = frame.FaceRect.Width;
rectangle2.Height = frame.FaceRect.Height;
Point rectPt2 = new Point();
rectPt2.X = frame.FaceRect.Left;
rectPt2.Y = frame.FaceRect.Top;
rectangle2.Location = (Point)rectPt2;
drawNum = 1;
break;
case 2:
rectangle3.Width = frame.FaceRect.Width;
rectangle3.Height = frame.FaceRect.Height;
Point rectPt3 = new Point();
rectPt3.X = frame.FaceRect.Left;
rectPt3.Y = frame.FaceRect.Top;
rectangle3.Location = (Point)rectPt3;
drawNum = 2;
break;
case 3:
rectangle4.Width = frame.FaceRect.Width;
rectangle4.Height = frame.FaceRect.Height;
Point rectPt4 = new Point();
rectPt4.X = frame.FaceRect.Left;
rectPt4.Y = frame.FaceRect.Top;
rectangle4.Location = (Point)rectPt4;
drawNum = 3;
break;
}
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest).Clone() as FaceTrackFrame;
}
}
public static void Write(this BinaryStreamWriter writer, SkeletonTrackingState value)
{
writer.Write(skeletonTrackingStateTable[value]);
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
//getting the Animation Unit Coefficients
this.AUs = frame.GetAnimationUnitCoefficients();
var jawLowerer = AUs[AnimationUnit.JawLower];
var browLower = AUs[AnimationUnit.BrowLower];
var browRaiser = AUs[AnimationUnit.BrowRaiser];
var lipDepressor = AUs[AnimationUnit.LipCornerDepressor];
var lipRaiser = AUs[AnimationUnit.LipRaiser];
var lipStretcher = AUs[AnimationUnit.LipStretcher];
//set up file for output
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt"))
{
file.WriteLine("FaceTrack Data, started recording at " + DateTime.Now.ToString("HH:mm:ss tt"));
}
//here is the algorithm to test different facial features
//BrowLower is messed up if you wear glasses, works if you don't wear 'em
//surprised
if ((jawLowerer < 0.25 || jawLowerer > 0.25) && browLower < 0)
{
System.Diagnostics.Debug.WriteLine("surprised");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": surprised");
file.WriteLine("JawLowerer: " + jawLowerer);
file.WriteLine("BrowLowerer: " + browLower);
}
}
//smiling
if (lipStretcher > 0.4 || lipDepressor<0)
{
System.Diagnostics.Debug.WriteLine("Smiling");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": smiling");
file.WriteLine("LipStretcher: " + lipStretcher);
}
}
//kissing face
if (lipStretcher < -0.75)
{
System.Diagnostics.Debug.WriteLine("kissing face");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": kissing face");
file.WriteLine("LipStretcher: " + lipStretcher);
}
}
//sad
if (browRaiser < 0 && lipDepressor>0)
{
System.Diagnostics.Debug.WriteLine("sad");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": sad");
file.WriteLine("LipCornerDepressor: " + lipDepressor);
file.WriteLine("OuterBrowRaiser: " + browRaiser);
}
}
//angry
if ((browLower > 0 && (jawLowerer > 0.25 || jawLowerer < -0.25)) ||
(browLower > 0 && lipDepressor > 0))
{
System.Diagnostics.Debug.WriteLine("angry");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": angry");
file.WriteLine("LipCornerDepressor: " + lipDepressor);
file.WriteLine("BrowLowerer: " + browLower);
file.WriteLine("JawLowerer: " + jawLowerer);
}
}
//System.Diagnostics.Debug.WriteLine(browLower);
this.facePoints = frame.GetProjected3DShape();
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this._skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this._skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this._faceTracker == null)
{
try
{
this._faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException e)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Console.WriteLine(Converter.EncodeError("FaceTracker.OnFrameReady - creating a new FaceTracker threw an InvalidOperationException: " + e.Message));
this._faceTracker = null;
}
}
if (this._faceTracker != null)
{
FaceTrackFrame frame = this._faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this._lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this._lastFaceTrackSucceeded)
{
var animUnits = frame.GetAnimationUnitCoefficients();
var pos = frame.Translation;
var rot = frame.Rotation;
var faceData = new FaceData();
faceData.Au0 = animUnits[0];
faceData.Au1 = animUnits[1];
faceData.Au2 = animUnits[2];
faceData.Au3 = animUnits[3];
faceData.Au4 = animUnits[4];
faceData.Au5 = animUnits[5];
faceData.PosX = pos.X;
faceData.PosY = pos.Y;
faceData.PosZ = pos.Z;
faceData.RotX = rot.X;
faceData.RotY = rot.Y;
faceData.RotZ = rot.Z;
var data = Converter.EncodeFaceTrackingData(faceData);
Console.WriteLine(data);
}
}
}
