private float[,] calcDist(EnumIndexableCollection <FeaturePoint, Vector3DF> fp)
{
int fpSize = fp.Count();
Vector3DF[] temp = new Vector3DF[fpSize];
float[,] fpRes = new float[fpSize, fpSize];
int k = 0;
foreach (Vector3DF p in fp)
{
temp[k] = p;
k++;
}
for (int i = 0; i < (fpSize - 1); i++)
{
for (int j = 0; j < (fpSize - 1); j++)
{
if (i == j)
{
}
// Do stuff to temp!
else
{
fpRes[i, j] = Math.Abs((float)Math.Sqrt((float)Math.Pow(temp[i].X - temp[j].X, 2) + (float)Math.Pow(temp[i].Y - temp[j].Y, 2) + (float)Math.Pow(temp[i].Z - temp[j].Z, 2)));
}
}
}
return(fpRes);
}
public Background(Vector3DF color, Vector3DF light)
{
byte[] shader_code =
Engine.Graphics.GraphicsDeviceType == GraphicsDeviceType.OpenGL
? Engine.File.CreateStaticFile("Shaders/BackEffect.glsl").Buffer
: Engine.Graphics.GraphicsDeviceType == GraphicsDeviceType.DirectX11
? Engine.File.CreateStaticFile("Shaders/BackEffect.hlsl").Buffer
: new byte[0];
// マテリアルを作成する
material = Engine.Graphics.CreateMaterial2D(
Engine.Graphics.CreateShader2D(
new UTF8Encoding().GetString(shader_code)
)
);
// ストップウォッチを作成する
stopwatch = new Stopwatch();
// マテリアル内の変数を設定する
material.SetVector2DF("resolution", Engine.WindowSize.To2DF());
material.SetVector3DF("color", color);
material.SetVector3DF("light", light);
// ストップウォッチを起動させる
stopwatch.Start();
}
public void ScaleXY(Vector3DF rotation, out int xResult, out int yResult)
{
double screenWidth = SystemParameters.PrimaryScreenWidth;
double y = ((SystemParameters.PrimaryScreenHeight / 30) * -rotation.X) +
(SystemParameters.PrimaryScreenHeight / 2);
double x = ((SystemParameters.PrimaryScreenWidth / 40) * -rotation.Y) +
(SystemParameters.PrimaryScreenWidth / 2);
if (x < 0)
{
x = 0;
}
else if (x > screenWidth - 1)
{
x = screenWidth - 1;
}
if (y < 0)
{
y = 0;
}
else if (y > SystemParameters.PrimaryScreenHeight - 1)
{
y = SystemParameters.PrimaryScreenHeight;
}
xResult = (int)x;
yResult = (int)y;
}
public System.Drawing.Point ScaleXY(Vector3DF rotation)
{
double screenWidth = Screen.PrimaryScreen.Bounds.Width;
double screenHeight = Screen.PrimaryScreen.Bounds.Height;
double y = ((screenHeight / (int)nudHeadToScreenRelationYHeight.Value) * -rotation.X) +
(screenHeight / 2);
double x = ((screenWidth / (int)nudHeadToScreenRelationXWidth.Value) * -rotation.Y) +
(screenWidth / 2);
if (x < 0)
{
x = 0;
}
else if (x > screenWidth - 1)
{
x = screenWidth - 1;
}
if (y < 0)
{
y = 0;
}
else if (y > screenHeight - 1)
{
y = screenHeight;
}
return(new System.Drawing.Point((int)x, (int)y));
}
/// <summary>
/// 回転角や位置情報をOSCにより送信
/// </summary>
/// <param name="rotHeadXYZ"></param>
/// <param name="posHeadXYZ"></param>
/// <param name="posHandRightXYZ"></param>
/// <param name="posHandLeftXYZ"></param>
private void SendOSCMessage(Vector3DF rotHeadXYZ, Vector3DF posHeadXYZ, Vector3DF posHandRightXYZ, Vector3DF posHandLeftXYZ)
{
// OSCメッセージ生成
OscMessage rotHeadOsc = new OscMessage(from, "/head_rot", null);
OscMessage posHeadOsc = new OscMessage(from, "/head_pos", null);
OscMessage posHandRightOsc = new OscMessage(from, "/hand_r", null);
OscMessage posHandLeftOsc = new OscMessage(from, "/hand_l", null);
// 情報の追加
rotHeadOsc.Append <float>(rotHeadXYZ.x);
rotHeadOsc.Append <float>(rotHeadXYZ.y);
rotHeadOsc.Append <float>(rotHeadXYZ.z);
posHeadOsc.Append <float>(posHeadXYZ.x);
posHeadOsc.Append <float>(posHeadXYZ.y);
posHeadOsc.Append <float>(posHeadXYZ.z);
posHandRightOsc.Append <float>(posHandRightXYZ.x);
posHandRightOsc.Append <float>(posHandRightXYZ.y);
posHandRightOsc.Append <float>(posHandRightXYZ.z);
posHandLeftOsc.Append <float>(posHandLeftXYZ.x);
posHandLeftOsc.Append <float>(posHandLeftXYZ.y);
posHandLeftOsc.Append <float>(posHandLeftXYZ.z);
// 送信
rotHeadOsc.Send(toExp);
posHeadOsc.Send(toExp);
posHandRightOsc.Send(toExp);
posHandLeftOsc.Send(toExp);
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor,
ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat,
short[] depthImage, Skeleton skeletonOfInterest) // <---------Skeleton data passed here *****************
{
// Here a skeletonOfInterest is available **********************************
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
// Here is where a skeletonOfInterest is available ***
// Call Track(), passing skeletonOfInterest
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
// Assign the facePoints
this.facePoints = frame.GetProjected3DShape();
// Gets the face data but does not draw the face or the skeleton
// This code gets the yaw, pitch and roll Capture it here
this.rotation = frame.Rotation; // <-- frame is a FaceTrackFrame
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
}
// For head 3D
// Copy rotation and position
this.rotation = frame.Rotation;
this.translation = frame.Translation;
}
}
private void UpdateMesh(FaceTrackFrame faceTrackingFrame)
{
EnumIndexableCollection <FeaturePoint, Vector3DF> shapePoints = faceTrackingFrame.Get3DShape();
EnumIndexableCollection <FeaturePoint, PointF> projectedShapePoints = faceTrackingFrame.GetProjected3DShape();
if (this.triangleIndices == null)
{
// Update stuff that doesn't change from frame to frame
this.triangleIndices = faceTrackingFrame.GetTriangles();
var indices = new Int32Collection(this.triangleIndices.Length * 3);
foreach (FaceTriangle triangle in this.triangleIndices)
{
indices.Add(triangle.Third);
indices.Add(triangle.Second);
indices.Add(triangle.First);
}
this.theGeometry.TriangleIndices = indices;
this.theGeometry.Normals = null; // Let WPF3D calculate these.
this.theGeometry.Positions = new Point3DCollection(shapePoints.Count);
this.theGeometry.TextureCoordinates = new PointCollection(projectedShapePoints.Count);
for (int pointIndex = 0; pointIndex < shapePoints.Count; pointIndex++)
{
this.theGeometry.Positions.Add(new Point3D());
this.theGeometry.TextureCoordinates.Add(new Point());
}
}
// Update the 3D model's vertices and texture coordinates
for (int pointIndex = 0; pointIndex < shapePoints.Count; pointIndex++)
{
Vector3DF point = shapePoints[pointIndex];
this.theGeometry.Positions[pointIndex] = new Point3D(point.X, point.Y, -point.Z);
PointF projected = projectedShapePoints[pointIndex];
this.theGeometry.TextureCoordinates[pointIndex] =
new Point(
projected.X / (double)this.colorImageWritableBitmap.PixelWidth,
projected.Y / (double)this.colorImageWritableBitmap.PixelHeight);
}
}
private System.Drawing.Point CalculateGaussianSmoothedXYPosition(FaceTrackFrame currentFaceFrame)
{
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < gaussFilter.Count - 1)
{
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
i++;
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
return(ScaleXY(rotationMedium));
}
public double getDist(Vector3DF point1, Vector3DF point2)
{
double X1 = point1.X;
double Y1 = point1.Y;
double Z1 = point1.Z;
double X2 = point2.X;
double Y2 = point2.Y;
double Z2 = point2.Z;
double deltaX = X1 - X2;
double deltaY = Y1 - Y2;
double deltaZ = Z1 - Z2;
double distance = Math.Sqrt(deltaX * deltaX + deltaY * deltaY + deltaZ * deltaZ);
return(distance);
}
public void Update(IPluginIO pin, DX11RenderContext context)
{
for (int i = 0; i < this.FOutput.SliceCount; i++)
{
bool update = this.FInvalidate;
DX11IndexedGeometry geom;
if (!this.FOutput[i].Contains(context))
{
geom = new DX11IndexedGeometry(context);
geom.InputLayout = Pos3Norm3Tex2Vertex.Layout;
geom.VertexSize = Pos3Norm3Tex2Vertex.VertexSize;
geom.HasBoundingBox = false;
geom.Topology = PrimitiveTopology.TriangleList;
var indexstream = new DataStream(KinectRuntime.FACE_INDICES.Length * 4, true, true);
indexstream.WriteRange(KinectRuntime.FACE_INDICES);
indexstream.Position = 0;
geom.IndexBuffer = new DX11IndexBuffer(context, indexstream, false, true);
geom.VerticesCount = this.FInFrame[i].GetProjected3DShape().Count;
var vbuffer = new SlimDX.Direct3D11.Buffer(context.Device, new BufferDescription()
{
BindFlags = BindFlags.VertexBuffer,
CpuAccessFlags = CpuAccessFlags.Write,
OptionFlags = ResourceOptionFlags.None,
SizeInBytes = geom.VerticesCount * geom.VertexSize,
Usage = ResourceUsage.Dynamic
});
geom.VertexBuffer = vbuffer;
this.FOutput[i][context] = geom;
update = true;
}
else
{
geom = this.FOutput[i][context];
}
if (update)
{
DataStream ds = geom.LockVertexBuffer();
ds.Position = 0;
EnumIndexableCollection <FeaturePoint, PointF> pp = this.FInFrame[i].GetProjected3DShape();
EnumIndexableCollection <FeaturePoint, Vector3DF> p = this.FInFrame[i].Get3DShape();
Vector3[] norms = new Vector3[p.Count];
int[] inds = KinectRuntime.FACE_INDICES;
int tricount = inds.Length / 3;
//Compute smoothed normals
for (int j = 0; j < tricount; j++)
{
int i1 = inds[j * 3];
int i2 = inds[j * 3 + 1];
int i3 = inds[j * 3 + 2];
Vector3 v1 = p[i1].SlimVector();
Vector3 v2 = p[i2].SlimVector();
Vector3 v3 = p[i3].SlimVector();
Vector3 faceEdgeA = v2 - v1;
Vector3 faceEdgeB = v1 - v3;
Vector3 norm = Vector3.Cross(faceEdgeB, faceEdgeA);
norms[i1] += norm; norms[i2] += norm; norms[i3] += norm;
}
for (int j = 0; j < geom.VerticesCount; j++)
{
Pos3Norm3Tex2Vertex vertex = new Pos3Norm3Tex2Vertex();
Vector3DF v = p[j];
vertex.Position = new Vector3(v.X, v.Y, v.Z);
vertex.Normals = Vector3.Normalize(norms[j]);
vertex.TexCoords = new Vector2(0, 0);
ds.Write <Pos3Norm3Tex2Vertex>(vertex);
}
geom.UnlockVertexBuffer();
}
}
}
public static Vector3 SlimVector(this Vector3DF v)
{
return(new Vector3(v.X, v.Y, v.Z));
}
public void CenterRotation()
{
this.initRotation = this.rotation;
}
private void UpdateMesh(FaceTrackFrame faceTrackingFrame)
{
//Console.Out.WriteLine(" ###################### In UpdateMesh ############################# ");
bool faceInCentre = true;
EnumIndexableCollection <FeaturePoint, Vector3DF> shapePoints = faceTrackingFrame.Get3DShape();
EnumIndexableCollection <FeaturePoint, PointF> projectedShapePoints = faceTrackingFrame.GetProjected3DShape();
if (this.triangleIndices == null)
{
// Update stuff that doesn't change from frame to frame
this.triangleIndices = faceTrackingFrame.GetTriangles();
var indices = new Int32Collection(this.triangleIndices.Length * 3);
foreach (FaceTriangle triangle in this.triangleIndices)
{
indices.Add(triangle.Third);
indices.Add(triangle.Second);
indices.Add(triangle.First);
}
this.theGeometry.TriangleIndices = indices;
this.theGeometry.Normals = null; // Let WPF3D calculate these.
this.theGeometry.Positions = new Point3DCollection(shapePoints.Count);
this.theGeometry.TextureCoordinates = new PointCollection(projectedShapePoints.Count);
for (int pointIndex = 0; pointIndex < shapePoints.Count; pointIndex++)
{
this.theGeometry.Positions.Add(new Point3D());
this.theGeometry.TextureCoordinates.Add(new Point());
}
}
// Update the 3D model's vertices and texture coordinates
for (int pointIndex = 0; pointIndex < shapePoints.Count; pointIndex++)
{
Vector3DF point = shapePoints[pointIndex];
this.theGeometry.Positions[pointIndex] = new Point3D(point.X, point.Y, -point.Z);
PointF projected = projectedShapePoints[pointIndex];
this.theGeometry.TextureCoordinates[pointIndex] =
new Point(
projected.X / (double)this.colorImageWritableBitmap.PixelWidth,
projected.Y / (double)this.colorImageWritableBitmap.PixelHeight);
// Console.Out.WriteLine("X = " + projected.X / (double)this.colorImageWritableBitmap.PixelWidth + "Y = " + projected.Y / (double)this.colorImageWritableBitmap.PixelHeight);
if (projected.X / (double)this.colorImageWritableBitmap.PixelWidth > .6 ||
projected.Y / (double)this.colorImageWritableBitmap.PixelHeight > .75)
{
faceInCentre = false;
}
}
if (faceInCentre)
{
// copyFaceImage();
FaceMesh tempMeshData = new FaceMesh();
tempMeshData.FaceViewport = viewport3d;
FaceMeshData = tempMeshData;
}
}
/// <summary>
/// Computes the absolute gap between two vectors just at numeric view
/// </summary>
/// <param name="vector1">The first vector.</param>
/// <param name="vector2">The second vector.</param>
/// <returns>The absolute gap</returns>
private static double GetAbsoluteGap(Vector3DF vector1, Vector3DF vector2)
{
return(Math.Abs(vector1.X - vector2.X) + Math.Abs(vector1.Y - vector2.Y) + Math.Abs(vector1.Z - vector2.Z));
}
/// <summary>
/// Computes the distance of two points
/// </summary>
/// <param name="point1">The first point.</param>
/// <param name="point2">The second point.</param>
/// <returns>The distance of two points</returns>
private static double GetDistance(Vector3DF point1, Vector3DF point2)
{
return(Math.Sqrt(((point1.X - point2.X) * (point1.X - point2.X))
+ ((point1.Y - point2.Y) * (point1.Y - point2.Y)) + ((point1.Z - point2.Z) * (point1.Z - point2.Z))));
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
///
public void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
//No Touchy
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
return;
}
if (faceTracker == null)
{
faceTracker = new FaceTracker(kinectSensor);
}
frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
//Touchy
//Assign Reference points
this.absfacePoints = frame.Get3DShape();
leftForehead = this.absfacePoints[FeaturePoint.TopLeftForehead];
rightForehead = this.absfacePoints[FeaturePoint.TopRightForehead];
jaw = this.absfacePoints[FeaturePoint.BottomOfChin];
faceRotationX = frame.Rotation.X;
faceRotationY = frame.Rotation.Y;
faceRotationZ = frame.Rotation.Z;
//Calculate Reference Points
foreheadReferencePointX = ((rightForehead.X - leftForehead.X) / 2);
foreheadReferencePointY = ((rightForehead.Y - leftForehead.Y) / 2);
foreheadReferencePointZ = ((rightForehead.Z - leftForehead.Z) / 2);
//Set Animation Units
AUCoeff = frame.GetAnimationUnitCoefficients();
jawLowererAU = AUCoeff[AnimationUnit.JawLower];
lipStretcherAU = AUCoeff[AnimationUnit.LipStretcher];
browRaiserAU = AUCoeff[AnimationUnit.BrowRaiser];
setJawData(jaw.Y, leftForehead.Y, rightForehead.Y, jawLowererAU, lipStretcherAU);
rotations = new float[5];
//set up matlab
matlab = new MLApp.MLApp();
matlab.Execute(@"cd C:\Users\Bala\Documents\MATLAB");
result = null;
//get rotation values
rotations[0] = faceRotationX;
rotations[1] = faceRotationY;
rotations[2] = faceRotationZ;
rotations[3] = jawLowererAU;
rotations[4] = lipStretcherAU;
//Set up GlovePie
OscPacket.LittleEndianByteOrder = false;
IPEndPoint myapp = new IPEndPoint(IPAddress.Loopback, 1944);
IPEndPoint glovepie = new IPEndPoint(IPAddress.Loopback, 1945);
Console.WriteLine(browRaiserAU);
matlab.Feval("nnW", 1, out result, rotations[0]);
object[] resW = result as object[];
nnoutput = (int)((float)resW[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 1;
}
else
{
result = null;
matlab.Feval("nnA", 1, out result, rotations[1]);
object[] resA = result as object[];
nnoutput = (int)((float)resA[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 2;
}
else
{
result = null;
matlab.Feval("nnS", 1, out result, rotations[0]);
object[] resS = result as object[];
nnoutput = (int)((float)resS[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 3;
}
else
{
result = null;
matlab.Feval("nnd", 1, out result, rotations[1]);
object[] resD = result as object[];
nnoutput = (int)((float)resD[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 4;
}
else
{
result = null;
matlab.Feval("nnLC", 1, out result, rotations[2]);
object[] resLC = result as object[];
nnoutput = (int)((float)resLC[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 5;
}
else
{
result = null;
matlab.Feval("nnRC", 1, out result, rotations[2]);
object[] resRC = result as object[];
nnoutput = (int)((float)resRC[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 6;
}
else
{
result = null;
if (jawLowererAU > 0.7)
{
commandtoSend = 7;
}
/*
* matlab.Feval("nnSpace", 1, out result, rotations[3]);
* object[] resSpace = result as object[];
* nnoutput = (int)((float)resSpace[0] + 0.5f);
* if (nnoutput == 1)
* {
* commandtoSend = 7;
* }*/
else
{
result = null;
if (browRaiserAU > 0.4)
{
commandtoSend = 8;
}
else
{
result = null;
commandtoSend = 0;
}
/*result = null;
* matlab.Feval("nnMiddle", 1, out result, lipStretcherAU);
* object[] resMiddle = result as object[];
* nnoutput = (int)((float)resMiddle[0] + 0.5f);
* if (nnoutput == 1)
* {
* commandtoSend = 8;
* }
* else
* {
* result = null;
* commandtoSend = 0;
* }*/
}
}
}
}
}
}
}
//Console.WriteLine("Iteration Complete");
switch (commandtoSend)
{
case 0:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 1:
msg = new OscMessage(myapp, "/move/w", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 2:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 3:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 4:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 5:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 6:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 7:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 8:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 10.0f);
msg.Send(glovepie);
break;
}
}
}
public void Update(Skeleton sk)
{
if (!AudioProvider.detectionActive)
{
TrackingSideTool.Get().SpeedRateDetectionCheckBox.IsChecked = true;
}
_up = false;
_complete = false;
_lookingDir = false;
frame++;
if (frame > NB_FRAME_MAX)
{
frame = 0;
start = false;
right = false;
left = false;
_up = true;
DrawingSheetAvatarViewModel.displayCustomText = String.Empty;
GestureRecognized?.Invoke(this, new EventArgs());
}
if (TrainingWithAvatarViewModel.Get().SkeletonList != null && TrainingWithAvatarViewModel.canBeInterrupted && !start)
{
DrawingSheetAvatarViewModel.displayCustomText = "Look at the direction of your raised arm then speak";
}
try
{
var face = KinectDevice.skeletonFaceTracker.facePoints3D;
Vector3DF rightEye = face.ElementAt(20);
Vector3DF leftEye = face.ElementAt(53);
Point3D handRight = new Point3D(sk.Joints[JointType.HandRight].Position);
Point3D handLeft = new Point3D(sk.Joints[JointType.HandLeft].Position);
Point3D elbowRight = new Point3D(sk.Joints[JointType.ElbowRight].Position);
Point3D elbowLeft = new Point3D(sk.Joints[JointType.ElbowLeft].Position);
Point3D shoulder = new Point3D(sk.Joints[JointType.ShoulderCenter].Position);
if (verbose)
{
Console.WriteLine("~~~~~~~~~~~~~~~~~~~~~~~~~");
Console.WriteLine("frame: " + frame);
Console.WriteLine("direction: " + (rightEye.Z - leftEye.Z));
Console.WriteLine("right: " + right);
Console.WriteLine("left: " + left);
}
if (Math.Abs(handRight.X - elbowRight.X) > 0.05 && handRight.Y > shoulder.Y)
{
if (rightEye.Z - leftEye.Z < -0.015)
{
if (AudioProvider.currentIntensity > AUDIO_LOUDNESS)
{
start = true;
right = true;
DrawingSheetAvatarViewModel.displayCustomText = "Do the same with the other arm";
if (left)
{
frame = 0;
start = false;
right = false;
left = false;
_complete = true;
DrawingSheetAvatarViewModel.displayCustomText = String.Empty;
GestureRecognized?.Invoke(this, new EventArgs());
}
}
}
else if (!right)
{
frame = 0;
start = false;
right = false;
left = false;
_lookingDir = true;
DrawingSheetAvatarViewModel.displayCustomText = String.Empty;
GestureRecognized?.Invoke(this, new EventArgs());
}
}
if (Math.Abs(handLeft.X - elbowLeft.X) > 0.05 && handLeft.Y > shoulder.Y)
{
if (rightEye.Z - leftEye.Z > 0.015)
{
if (AudioProvider.currentIntensity > AUDIO_LOUDNESS)
{
start = true;
left = true;
DrawingSheetAvatarViewModel.displayCustomText = "Do the same with the other arm";
if (right)
{
frame = 0;
start = false;
right = false;
left = false;
_complete = true;
DrawingSheetAvatarViewModel.displayCustomText = String.Empty;
GestureRecognized?.Invoke(this, new EventArgs());
}
}
}
else if (!left)
{
frame = 0;
start = false;
right = false;
left = false;
_lookingDir = true;
DrawingSheetAvatarViewModel.displayCustomText = String.Empty;
GestureRecognized?.Invoke(this, new EventArgs());
}
}
}
catch (Exception e) //the kinect don't catch the face
{
//System.Windows.Forms.MessageBox.Show("the kinect don't catch the face !");
Console.Error.WriteLine(e);
}
}
private void OnAllFramesReady(object sender, Microsoft.Kinect.AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.SkeletonData);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
//Idea: Separate Eye-Parts of Color Image
//Use learning Algorithm for right and left eye
//Detect blink on separated parts of color Image
//colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values
if (activeSkeleton != null)
{
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser];
float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower];
tbBrowLowerer.Text = browLowererValue.ToString();
tbBrowRaiser.Text = browRaiserValue.ToString();
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
pbLeftEye.Image = leftEye;
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
pbRightEye.Image = rightEye;
//Wende Kantenfilter auf die beiden Augen an.
double dxRight;
double dyRight;
double dxLeft;
double dyLeft;
if (rightEye != null && leftEye != null)
{
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out dxRight, out dyRight);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out dxLeft, out dyLeft);
//If Face is rotated, move Mouse
if (headRotationHistory.Count > filterLength && currentFaceFrame.TrackSuccessful)
{
int x = 0;
int y = 0;
//Method 1: Ohne Glättung
//ScaleXY(currentFaceFrame.Rotation, out x, out y);
//MouseControl.Move(x, y);
////Method 2: Glättung über die letzten x Bilder:
//int i = 0;
//Vector3DF rotationMedium = new Vector3DF();
//while (i < 10 && headRotationHistory.Count - 1 > i)
//{
// i++;
// rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X;
// rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y;
//}
//rotationMedium.X = rotationMedium.X / i;
//rotationMedium.Y = rotationMedium.Y / i;
//ScaleXY(rotationMedium, out x, out y);
//MouseControl.Move(x, y);
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < filterLength - 1)
{
i++;
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
ScaleXY(rotationMedium, out x, out y);
MouseControl.Move(x, y);
//Method 4: Quadratische Glättung
//double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y));
//double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X));
//if (deltaX < 0)
// deltaX = -Math.Pow(deltaX, 2) * 4;
//else
// deltaX = Math.Pow(deltaX, 2) * 4;
//if (deltaY < 0)
// deltaY = -Math.Pow(deltaY, 2) * 5;
//else
// deltaY = Math.Pow(deltaY, 2) * 5;
//MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY));
}
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
}
}
catch (Exception e)
{
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData); // An array of skeletons
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
// Pass the skeleton to the skeletonFaceTracker - see that class below
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
this.rotation = skeletonFaceTracker.rotation;
// Draw the skeleton here...
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
File.AppendAllText("mouseLog.txt", DateTime.Now + " - All Kinect frames ready.\n");
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Color- depth or Skeletonframe is null. Aborting Frame.\n");
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.SkeletonData);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
//Idea: Separate Eye-Parts of Color Image
//Use learning Algorithm for right and left eye
//Detect blink on separated parts of color Image
//colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values
if (activeSkeleton != null)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Skeleton is there. Trying to find face.\n");
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
if (currentFaceFrame.TrackSuccessful)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Recognized face successfully.\n");
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Couldn't find face in frame.\n");
}
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
//this.pbRight.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye));
//
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
Bitmap leftEye2 = null;
Bitmap rightEye2 = null;
if (leftEye != null)
{
leftEye2 = new Bitmap(leftEye);
}
if (rightEye != null)
{
rightEye2 = new Bitmap(rightEye);
}
// System.Delegate d = new MethodInvoker(SetPictures));
// this.Invoke(SetPictures, leftEye);
//pbRight.Image = rightEye;
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbLeft.Image = rightEye2));
this.pbLeft.BeginInvoke((MethodInvoker)(() => this.pbRight.Image = leftEye2));
// this.Invoke(new MethodInvoker(SetPictures));
//Wende Kantenfilter auf die beiden Augen an.
if (rightEye != null && leftEye != null)
{
Dictionary <string, int> angleCount;
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out angleCount);
bool rightEyeClosed = IsEyeClosed(angleCount);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out angleCount);
bool leftEyeClosed = IsEyeClosed(angleCount);
// pbLeftFaltung.Image = edgePicLeft;
// pbRightFaltung.Image = edgePicRight;
if (rightEyeClosedHistory.Count > 100)
{
rightEyeClosedHistory.RemoveAt(0);
}
if (leftEyeClosedHistory.Count > 100)
{
leftEyeClosedHistory.RemoveAt(0);
}
leftEyeClosedHistory.Add(leftEyeClosed);
rightEyeClosedHistory.Add(rightEyeClosed);
//If Face is rotated, move Mouse
if (headRotationHistory.Count > gaussFilter.Count - 1 && leftEyeClosedHistory.Count > nudConvolutionFilterLength.Value && currentFaceFrame.TrackSuccessful)
{
int x = 0;
int y = 0;
float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser];
float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower];
float mouthOpenValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.JawLower];
if (browRaiserHistory.Count >= 100)
{
browRaiserHistory.RemoveAt(0);
browLowererHistory.RemoveAt(0);
mouthOpenHistory.RemoveAt(0);
}
browLowererHistory.Add(browLowererValue);
browRaiserHistory.Add(browRaiserValue);
mouthOpenHistory.Add(mouthOpenValue);
//Method 1: Ohne Glättung
//ScaleXY(currentFaceFrame.Rotation, out x, out y);
//MouseControl.Move(x, y);
////Method 2: Glättung über die letzten x Bilder:
//int i = 0;
//Vector3DF rotationMedium = new Vector3DF();
//while (i < 10 && headRotationHistory.Count - 1 > i)
//{
// i++;
// rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X;
// rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y;
//}
//rotationMedium.X = rotationMedium.X / i;
//rotationMedium.Y = rotationMedium.Y / i;
//ScaleXY(rotationMedium, out x, out y);
//MouseControl.Move(x, y);
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < gaussFilter.Count - 1)
{
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
i++;
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
ScaleXY(rotationMedium, out x, out y);
//Method 4: Quadratische Glättung
//double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y));
//double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X));
//if (deltaX < 0)
// deltaX = -Math.Pow(deltaX, 2) * 4;
//else
// deltaX = Math.Pow(deltaX, 2) * 4;
//if (deltaY < 0)
// deltaY = -Math.Pow(deltaY, 2) * 5;
//else
// deltaY = Math.Pow(deltaY, 2) * 5;
//MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY));
//Check for right, left or Double Click
//1. Check if there was already a click 20 Frames ago, or if Drag & Drop is active
if (clickDelay > nudClickDelay.Value && !pointNClickActive)
{
//2. If not, calculate mean values of dy's last 16 Frames
if (CalculateMeanConvolutionValues())
{
clickDelay = 0;
}
else
{
//Else check for open Mouth
if (mouthOpenValue > (float)nudMouthOpenStartThreshold.Value && mouthOpenHistory[mouthOpenHistory.Count - 2] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 3] > (float)nudMouthOpenConfirmation.Value && mouthOpenHistory[mouthOpenHistory.Count - 4] > (float)nudMouthOpenConfirmation.Value)
{
MouseControl.Move(mousePositionHistory[mousePositionHistory.Count - 4].X, mousePositionHistory[mousePositionHistory.Count - 4].Y);
this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y)));
//lbAction.Items.Add("Left Mouse Down on X: " + mousePositionHistory[mousePositionHistory.Count - 4].X + " Y: " + mousePositionHistory[mousePositionHistory.Count - 4].Y);
MouseControl.MouseDownLeft();
pointNClickActive = true;
clickDelay = 0;
}
}
}
else if (pointNClickActive)
{
if (mouthOpenValue < (float)nudMouthOpenEndThreshold.Value)
{
this.lbAction.Invoke((MethodInvoker)(() => this.lbAction.Items.Add("Left Mouse Up on X: " + x + " Y: " + y)));
MouseControl.MouseUpLeft();
pointNClickActive = false;
clickDelay = 0;
}
}
MouseControl.Move(x, y);
if (browLowererValue > (float)nudBrowLowererStartThreshold.Value)
{
MouseControl.ScrollDown((int)(-browLowererValue * (int)nudScrollMultiplierDown.Value));
}
if (browRaiserValue > (float)nudBrowRaiserStartThreshold.Value)
{
MouseControl.ScrollDown((int)(browRaiserValue * (int)nudScrollMultiplierUp.Value));
}
if (mousePositionHistory.Count > 100)
{
mousePositionHistory.RemoveAt(0);
}
mousePositionHistory.Add(new Microsoft.Kinect.Toolkit.FaceTracking.Point(x, y));
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face and eyes successfully tracked.\n");
}
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Face recognized but couldn't find eye in face.\n");
}
clickDelay++;
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
else
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Active Skeleton is null. Couldn't analyze frame.\n");
}
}
catch (Exception e)
{
File.AppendAllText("mouseLog.txt", DateTime.Now + " - Error during frame analyzation.\n" + e.ToString());
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
