public Option <HandInputEvent> Update(SkeletonFrame sf, byte[] cf)
{
if (sf != null)
{
if (skeletonData == null || skeletonData.Length != sf.SkeletonArrayLength)
{
skeletonData = new Skeleton[sf.SkeletonArrayLength];
}
sf.CopySkeletonDataTo(skeletonData);
var trackedIndex = SkeletonUtil.FirstTrackedSkeletonIndex(skeletonData);
if (trackedIndex >= 0)
{
var skeleton = skeletonData[trackedIndex];
var handRight = SkeletonUtil.GetJoint(skeleton, JointType.HandRight);
var handLeft = SkeletonUtil.GetJoint(skeleton, JointType.HandLeft);
var handRightPos = coordMapper.MapSkeletonPointToDepthPoint(handRight.Position,
DepthImageFormat.Resolution640x480Fps30);
var handLeftPos = coordMapper.MapSkeletonPointToDepthPoint(handLeft.Position,
DepthImageFormat.Resolution640x480Fps30);
return(new Some <HandInputEvent>(new HandInputEvent(handLeftPos, handRightPos)));
}
}
return(new None <HandInputEvent>());
}
private double GetWaistWidth(Skeleton skel)
{
/* Get the points of the right and left shoulders */
CoordinateMapper coordMapper = new CoordinateMapper(sensorChooser.Kinect);
DepthImagePoint rightHip = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.HipRight].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint leftHip = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.HipLeft].Position, DepthImageFormat.Resolution640x480Fps30);
if (rightHip == null || leftHip == null)
{
throw new ArgumentNullException("Could not find one or both shoulders");
}
return(Math.Sqrt(Math.Pow(Math.Abs(rightHip.X - leftHip.X), 2) + Math.Pow(Math.Abs(rightHip.Y - leftHip.Y), 2)));
}
public void MapCameraPointsToDepthSpace(Vector3[] cameraPoints, Vector2[] depthPoints)
{
for (int i = 0; i < cameraPoints.Length && i < depthPoints.Length; i++)
{
var colorSpacePoint =
_coordinateMapper.MapSkeletonPointToDepthPoint(new SkeletonPoint()
{
X = cameraPoints[i].X,
Y = cameraPoints[i].Y,
Z = cameraPoints[i].Z
}, DepthImageFormat.Resolution640x480Fps30);
depthPoints[i] = new Vector2(colorSpacePoint.X, colorSpacePoint.Y);
}
}
/// <summary>
/// Maps a 3D skeleton point to a 2D vector.
/// </summary>
/// <param name="sensor">The Kinect sensor.</param>
/// <param name="position">The skeleton point to map.</param>
/// <param name="coordinateMapper">The coordinate mapper.</param>
/// <returns>The 2D mapped position.</returns>
public static Vector2 Convert(KinectSensor sensor, SkeletonPoint position, CoordinateMapper coordinateMapper)
{
float width = 0;
float height = 0;
float x = 0;
float y = 0;
if (sensor.ColorStream.IsEnabled)
{
var colorPoint = coordinateMapper.MapSkeletonPointToColorPoint(position, sensor.ColorStream.Format);
x = colorPoint.X;
y = colorPoint.Y;
switch (sensor.ColorStream.Format)
{
case ColorImageFormat.RawYuvResolution640x480Fps15:
case ColorImageFormat.RgbResolution640x480Fps30:
case ColorImageFormat.YuvResolution640x480Fps15:
width = 640;
height = 480;
break;
case ColorImageFormat.RgbResolution1280x960Fps12:
width = 1280;
height = 960;
break;
}
}
else if (sensor.DepthStream.IsEnabled)
{
var depthPoint = coordinateMapper.MapSkeletonPointToDepthPoint(position, sensor.DepthStream.Format);
x = depthPoint.X;
y = depthPoint.Y;
switch (sensor.DepthStream.Format)
{
case DepthImageFormat.Resolution80x60Fps30:
width = 80;
height = 60;
break;
case DepthImageFormat.Resolution320x240Fps30:
width = 320;
height = 240;
break;
case DepthImageFormat.Resolution640x480Fps30:
width = 640;
height = 480;
break;
}
}
else
{
width = 1;
height = 1;
}
return(new Vector2(x / width, y / height));
}
/// <summary>
/// Maps a SkeletonPoint to lie within our render space and converts to Point
/// </summary>
/// <param name="skelpoint">point to map</param>
/// <returns>mapped point</returns>
private Point SkeletonPointToScreen(SkeletonPoint skelpoint)
{
// Convert point to depth space.
// We are not using depth directly, but we do want the points in our 640x480 output resolution.
DepthImagePoint depthPoint = _mapper.MapSkeletonPointToDepthPoint(skelpoint, DepthImageFormat.Resolution640x480Fps30);
return(new Point(depthPoint.X, depthPoint.Y));
}
void tryToMoveBird(Skeleton first, AllFramesReadyEventArgs e, int skeletonID)
{
using (DepthImageFrame depth = e.OpenDepthImageFrame())
{
if (depth == null ||
kinectSensorChooser1.Kinect == null)
{
return;
}
CoordinateMapper cm = new CoordinateMapper(kinectSensorChooser1.Kinect);
DepthImagePoint rightWristDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.WristRight].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightElbowDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.ElbowRight].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightHandDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30);
if (isArmHorizontal(rightWristDepthPoint, rightElbowDepthPoint))
{
frameCounter++;
//System.Diagnostics.Debug.Write("\nframeCounter = " + frameCounter);
if (frameCounter >= 30)
{
fromUser = true;
moveBirdToUser(birdHand, rightHandDepthPoint);
birdHandFlying.Visibility = Visibility.Hidden;
birdHand.Visibility = Visibility.Visible;
}
}
else
{
//System.Diagnostics.Debug.Write("\nif de RESET");
if (fromUser == true)
{
Random rnd = new Random();
XtoReset = rnd.Next(30, 610); // [x1, x2[
YtoReset = rnd.Next(30, 40); // [y1, y2[
fromUser = false;
resetBird(birdHand);
birdHandFlying.Visibility = Visibility.Visible;
birdHand.Visibility = Visibility.Hidden;
}
frameCounter = 0;
}
}
}
/// <summary>
/// Maps a 3D skeleton point to a 2D vector.
/// </summary>
/// <param name="sensor">The Kinect sensor.</param>
/// <param name="position">The skeleton point to map.</param>
/// <param name="coordinateMapper">The coordinate mapper.</param>
/// <returns>The 2D mapped position.</returns>
public static Vector2 Convert(KinectSensor sensor, SkeletonPoint position, CoordinateMapper coordinateMapper)
{
float width = 0;
float height = 0;
float x = 0;
float y = 0;
if (sensor.ColorStream.IsEnabled)
{
var colorPoint = coordinateMapper.MapSkeletonPointToColorPoint(position, sensor.ColorStream.Format);
x = colorPoint.X;
y = colorPoint.Y;
switch (sensor.ColorStream.Format)
{
case ColorImageFormat.RawYuvResolution640x480Fps15:
case ColorImageFormat.RgbResolution640x480Fps30:
case ColorImageFormat.YuvResolution640x480Fps15:
width = 640;
height = 480;
break;
case ColorImageFormat.RgbResolution1280x960Fps12:
width = 1280;
height = 960;
break;
}
}
else if (sensor.DepthStream.IsEnabled)
{
var depthPoint = coordinateMapper.MapSkeletonPointToDepthPoint(position, sensor.DepthStream.Format);
x = depthPoint.X;
y = depthPoint.Y;
switch (sensor.DepthStream.Format)
{
case DepthImageFormat.Resolution80x60Fps30:
width = 80;
height = 60;
break;
case DepthImageFormat.Resolution320x240Fps30:
width = 320;
height = 240;
break;
case DepthImageFormat.Resolution640x480Fps30:
width = 640;
height = 480;
break;
}
}
else
{
width = 1;
height = 1;
}
return new Vector2(x / width, y / height);
}
/// <summary>
/// Map PointSkeleton3D to PointDepth3D
/// </summary>
/// <param name="pointSkleton3D"></param>
/// <param name="depthImageFormat"></param>
/// <returns></returns>
public PointDepth3D MapSkeletonPointToDepthPoint(PointSkeleton3D pointSkleton3D, DepthImageFormat depthImageFormat)
{
SkeletonPoint point = new SkeletonPoint();
point.X = pointSkleton3D.X;
point.Y = pointSkleton3D.Y;
point.Z = pointSkleton3D.Z;
return(new PointDepth3D(mapper.MapSkeletonPointToDepthPoint(point, depthImageFormat)));
}
public static Vector2 Convert(CoordinateMapper mapper, SkeletonPoint position,
Object format = null)
{
float width = 0;
float height = 0;
float x = 0;
float y = 0;
if (format != null) {
if (format is ColorImageFormat) {
var colorFormat = (ColorImageFormat)format;
var colorPoint = mapper.MapSkeletonPointToColorPoint(position, colorFormat);
x = colorPoint.X;
y = colorPoint.Y;
switch (colorFormat) {
case ColorImageFormat.RawYuvResolution640x480Fps15:
case ColorImageFormat.RgbResolution640x480Fps30:
case ColorImageFormat.YuvResolution640x480Fps15:
width = 640;
height = 480;
break;
case ColorImageFormat.RgbResolution1280x960Fps12:
width = 1280;
height = 960;
break;
}
} else if (format is DepthImageFormat) {
var depthFormat = (DepthImageFormat)format;
var depthPoint = mapper.MapSkeletonPointToDepthPoint(position, depthFormat);
x = depthPoint.X;
y = depthPoint.Y;
switch (depthFormat) {
case DepthImageFormat.Resolution80x60Fps30:
width = 80;
height = 60;
break;
case DepthImageFormat.Resolution320x240Fps30:
width = 320;
height = 240;
break;
case DepthImageFormat.Resolution640x480Fps30:
width = 640;
height = 480;
break;
}
} else {
width = 1;
height = 1;
}
}
return new Vector2(x / width, y / height);
}
public void DrawBone(Joint first, Joint second, Brush brush, double thickness)
{
if (first.TrackingState == JointTrackingState.NotTracked || second.TrackingState == JointTrackingState.NotTracked)
{
return;
}
Point firstPoint = new Point(_ratioX, _ratioY);
Point secondPoint = new Point(_ratioX, _ratioY);
switch (FrameType)
{
case VisualizationMode.Color:
{
ColorImagePoint colorFirstPoint = CoordinateMapper.MapSkeletonPointToColorPoint(first.Position, ColorImageFormat.RgbResolution640x480Fps30);
firstPoint.X *= float.IsInfinity(colorFirstPoint.X) ? 0.0 : colorFirstPoint.X;
firstPoint.Y *= float.IsInfinity(colorFirstPoint.Y) ? 0.0 : colorFirstPoint.Y;
ColorImagePoint colorSecondPoint = CoordinateMapper.MapSkeletonPointToColorPoint(second.Position, ColorImageFormat.RgbResolution640x480Fps30);
secondPoint.X *= float.IsInfinity(colorSecondPoint.X) ? 0.0 : colorSecondPoint.X;
secondPoint.Y *= float.IsInfinity(colorSecondPoint.Y) ? 0.0 : colorSecondPoint.Y;
}
break;
case VisualizationMode.Depth:
case VisualizationMode.Infrared:
{
DepthImagePoint depthFirstPoint = CoordinateMapper.MapSkeletonPointToDepthPoint(first.Position, DepthImageFormat.Resolution320x240Fps30);
firstPoint.X *= float.IsInfinity(depthFirstPoint.X) ? 0.0 : depthFirstPoint.X;
firstPoint.Y *= float.IsInfinity(depthFirstPoint.Y) ? 0.0 : depthFirstPoint.Y;
DepthImagePoint depthSecondPoint = CoordinateMapper.MapSkeletonPointToDepthPoint(second.Position, DepthImageFormat.Resolution320x240Fps30);
secondPoint.X *= float.IsInfinity(depthSecondPoint.X) ? 0.0 : depthSecondPoint.X;
secondPoint.Y *= float.IsInfinity(depthSecondPoint.Y) ? 0.0 : depthSecondPoint.Y;
}
break;
default:
break;
}
Line line = new Line
{
Tag = TAG,
X1 = firstPoint.X,
Y1 = firstPoint.Y,
X2 = secondPoint.X,
Y2 = secondPoint.Y,
StrokeThickness = thickness,
Stroke = brush
};
canvas.Children.Add(line);
}
private Point GetJointPoint(Joint joint)
{
CoordinateMapper cm = new CoordinateMapper(kinectDevice);
DepthImagePoint point = cm.MapSkeletonPointToDepthPoint(joint.Position, this.KinectDevice.DepthStream.Format);
//ColorImagePoint point = cm.MapSkeletonPointToColorPoint(joint.Position, this.KinectDevice.ColorStream.Format);
point.X *= (int)this.LayoutRoot.ActualWidth / KinectDevice.DepthStream.FrameWidth;
point.Y *= (int)this.LayoutRoot.ActualHeight / KinectDevice.DepthStream.FrameHeight;
return(new Point(point.X, point.Y));
}
//DEPTH_END
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e)
{
using (DepthImageFrame depth = e.OpenDepthImageFrame())
{
if (depth == null || kinectSensorChooser1.Kinect == null)
{
return;
}
CoordinateMapper cm = new CoordinateMapper(kinectSensorChooser1.Kinect);
DepthImagePoint headDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.Head].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint leftDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandLeft].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30);
ColorImagePoint headColorPoint = cm.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, headDepthPoint, ColorImageFormat.RgbResolution640x480Fps30);
ColorImagePoint leftColorPoint = cm.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, leftDepthPoint, ColorImageFormat.RgbResolution640x480Fps30);
ColorImagePoint rightColorPoint = cm.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, rightDepthPoint, ColorImageFormat.RgbResolution640x480Fps30);
//=> CameraPosition(headImage, headColorPoint);
//=> CameraPosition(leftEllipse, leftColorPoint);
//=> CameraPosition(rightEllipse, rightColorPoint);
}
}
private void drawPath(DrawingContext dc, CoordinateMapper cm)
{
const double length = 100;
DepthImagePoint p = cm.MapSkeletonPointToDepthPoint(refPoint, DepthImageFormat.Resolution640x480Fps30);
double rads = trackedAngle * Math.PI / 180;
double dx = length * Math.Cos(rads);
double dy = -length *Math.Sin(rads);
Point endPoint = new Point(p.X + dx, p.Y + dy);
dc.DrawLine(arrowPen, new Point(p.X, p.Y), endPoint);
dc.DrawEllipse(Brushes.Red, null, endPoint, 10.0, 10.0);
}
static void AllFramesReady(Object sender, AllFramesReadyEventArgs e)
{
// Console.WriteLine("AllFramesReady() called");
if (e != null)
{
SkeletonFrame sFrame = e.OpenSkeletonFrame();
CoordinateMapper cm = new CoordinateMapper(sensor);
if (sFrame != null)
{
// Console.WriteLine("Got Skeleton");
Skeleton[] skeletons = new Skeleton[sFrame.SkeletonArrayLength];
sFrame.CopySkeletonDataTo(skeletons);
SkeletonPoint sLoc = new SkeletonPoint();
foreach (Skeleton s in skeletons)
{
if (s.TrackingState == SkeletonTrackingState.Tracked)
{
sLoc = s.Joints[JointType.Head].Position;
DepthFrameReady(sender, e.OpenDepthImageFrame(), e.OpenColorImageFrame(), sLoc);
// Console.WriteLine("Head coordinates: " + sLoc.X + "," + sLoc.Y + "," + sLoc.Z);
ColorImagePoint cLoc = cm.MapSkeletonPointToColorPoint(sLoc, ColorImageFormat.RgbResolution640x480Fps30);
DepthImagePoint dLoc = cm.MapSkeletonPointToDepthPoint(sLoc, DepthImageFormat.Resolution640x480Fps30);
Console.WriteLine("Head coordinates: " + dLoc.X + "," + dLoc.Y);
}
}
sFrame.Dispose();
}
if (block == false)
{
dp = new DepthImagePoint();
cp = new ColorImagePoint();
cp.X = 320;
cp.Y = 240;
dp.X = 320;
dp.Y = 240;
////cp = cLoc;
//cp = cm.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, dp, ColorImageFormat.RgbResolution640x480Fps30);
Console.WriteLine(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + cp.X + " " + cp.Y);
//dp = dLoc;
}
// ColorImagePoint cLoc = sensor.MapSkeletonPointToColor(sLoc, e.OpenColorImageFrame().Format);
// nearX = dLoc.X;
// nearY = dLoc.Y;
nearX = dp.X;
nearY = dp.Y;
ImageFrameReady(sender, e.OpenColorImageFrame());
}
}
//Serializes an array of Kinect skeletons into an array of JSON skeletons.The Kinect skeletons.The coordinate mapper.Mode (color or depth)
public static string Serialize(this List <Skeleton> skeletons, CoordinateMapper mapper, Mode mode)
{
JSONSkeletonCollection jsonSkeletons = new JSONSkeletonCollection {
Skeletons = new List <JSONSkeleton>()
};
foreach (var skeleton in skeletons)
{
JSONSkeleton jsonSkeleton = new JSONSkeleton
{
ID = skeleton.TrackingId.ToString(),
Joints = new List <JSONJoint>()
};
foreach (Joint joint in skeleton.Joints)
{
Point point = new Point();
switch (mode)
{
case Mode.Color:
ColorImagePoint colorPoint = mapper.MapSkeletonPointToColorPoint(joint.Position, ColorImageFormat.RgbResolution640x480Fps30);
point.X = colorPoint.X;
point.Y = colorPoint.Y;
break;
case Mode.Depth:
DepthImagePoint depthPoint = mapper.MapSkeletonPointToDepthPoint(joint.Position, DepthImageFormat.Resolution640x480Fps30);
point.X = depthPoint.X;
point.Y = depthPoint.Y;
break;
default:
break;
}
jsonSkeleton.Joints.Add(new JSONJoint
{
Name = joint.JointType.ToString().ToLower(),
X = joint.Position.X,
Y = joint.Position.Y,
Z = joint.Position.Z
});
}
jsonSkeletons.Skeletons.Add(jsonSkeleton);
}
return(Serialize(jsonSkeletons));
}
internal void ProcessFrame(CoordinateMapper mapper, Skeleton skeletonOfInterest, DepthImageFormat depthImageFormat)
{
_joints.Clear();
if (skeletonOfInterest != null)
{
var size = FormatHelper.GetDepthSize(depthImageFormat);
var depthWidth = (int)size.Width;
var headJoint = skeletonOfInterest.Joints[JointType.Head];
var neckJoint = skeletonOfInterest.Joints[JointType.ShoulderCenter];
var _headPoint = mapper.MapSkeletonPointToDepthPoint(headJoint.Position, depthImageFormat);
var _neckPoint = mapper.MapSkeletonPointToDepthPoint(neckJoint.Position, depthImageFormat);
_headPoint.X = depthWidth - _headPoint.X;
_neckPoint.X = depthWidth - _neckPoint.X;
_joints.Add(_headPoint);
_joints.Add(_neckPoint);
}
RaiseFrameUpdated();
}
/// <summary>
/// Depth Frame Ready
/// Kinect tracks skeletons, localize the Skeleton Hand Joint Position
/// Map Skeleton point given by Kinect to PointDepth3D , Add result to HandTracker's List buffer for computing every frame
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
void sensor_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
{
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame != null)
{
if (skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
skeletonFrame.CopySkeletonDataTo(skeletonData);
int personCount = 0;
foreach (Skeleton sk in skeletonData)
{
if (sk.TrackingState == SkeletonTrackingState.Tracked)
{
//Skeleton point map to depth point given by Kinect
CoordinateMapper mapper = new CoordinateMapper(this.sensor);
DepthImagePoint left = mapper.MapSkeletonPointToDepthPoint(
sk.Joints[JointType.HandLeft].Position,
this.sensor.DepthStream.Format);
DepthImagePoint right = mapper.MapSkeletonPointToDepthPoint(
sk.Joints[JointType.HandRight].Position,
this.sensor.DepthStream.Format);
rightWrist = new PointSkeleton3D(sk.Joints[JointType.WristRight].Position);
rightHand = new PointSkeleton3D(sk.Joints[JointType.HandRight].Position);
rightElbow = new PointSkeleton3D(sk.Joints[JointType.ElbowRight].Position);
personCount++;
}
}
skeletonReadyFlag = (personCount == 0 ? false : true);
}
}
}
public void DrawJoint(Joint joint, Brush brush, double radius)
{
if (joint.TrackingState == JointTrackingState.NotTracked)
{
return;
}
Point point = new Point(_ratioX, _ratioY);
switch (FrameType)
{
case VisualizationMode.Color:
{
ColorImagePoint colorPoint = CoordinateMapper.MapSkeletonPointToColorPoint(joint.Position, ColorImageFormat.RgbResolution640x480Fps30);
point.X *= float.IsInfinity(colorPoint.X) ? 0.0 : colorPoint.X;
point.Y *= float.IsInfinity(colorPoint.Y) ? 0.0 : colorPoint.Y;
}
break;
case VisualizationMode.Depth:
case VisualizationMode.Infrared:
{
DepthImagePoint depthPoint = CoordinateMapper.MapSkeletonPointToDepthPoint(joint.Position, DepthImageFormat.Resolution320x240Fps30);
point.X *= float.IsInfinity(depthPoint.X) ? 0.0 : depthPoint.X;
point.Y *= float.IsInfinity(depthPoint.Y) ? 0.0 : depthPoint.Y;
}
break;
default:
break;
}
Ellipse ellipse = new Ellipse
{
Tag = TAG,
Width = radius,
Height = radius,
Fill = brush
};
Canvas.SetLeft(ellipse, point.X - ellipse.Width / 2);
Canvas.SetTop(ellipse, point.Y - ellipse.Height / 2);
canvas.Children.Add(ellipse);
}
/// <summary>
/// スケルトン(3D)座標を2D座標に変換する
/// </summary>
/// <param name="position"></param>
/// <returns></returns>
private Point SkeletonToPiont(SkeletonPoint position)
{
Point point;
Point _2d;
if (SkeletonConvert == Skeleton2DPoint.Color)
{
var p = mapper.MapSkeletonPointToColorPoint(position, kinect.ColorStream.Format);
point = new Point(p.X, p.Y);
_2d = new Point(kinect.ColorStream.FrameWidth, kinect.ColorStream.FrameHeight);
}
else
{
var p = mapper.MapSkeletonPointToDepthPoint(position, kinect.DepthStream.Format);
point = new Point(p.X, p.Y);
_2d = new Point(kinect.DepthStream.FrameWidth, kinect.DepthStream.FrameHeight);
}
return(KinectUtility.ScaleTo(point, _2d, new Point(canvas.ActualWidth, canvas.ActualHeight)));
}
private Point SkeletonPointToScreen(SkeletonPoint skelpoint, CoordinateMapper cm)
{
DepthImagePoint depthPoint = cm.MapSkeletonPointToDepthPoint(skelpoint, DepthImageFormat.Resolution640x480Fps30);
return(new Point(depthPoint.X, depthPoint.Y));
}
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e)
{
using (DepthImageFrame depth = e.OpenDepthImageFrame())
{
if (depth == null ||
kinectSensorChooser1.Kinect == null)
{
return;
}
//Map a joint location to a point on the depth map
//head
CoordinateMapper mapper = new CoordinateMapper(sensor);
DepthImagePoint headDepthPoint =
mapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.Head].Position, DepthImageFormat.Resolution640x480Fps30);
//left hand
DepthImagePoint leftDepthPoint =
mapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandLeft].Position, DepthImageFormat.Resolution640x480Fps30);
//right hand
DepthImagePoint rightDepthPoint =
mapper.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30);
//Map a depth point to a point on the color image
//head
ColorImagePoint headColorPoint =
mapper.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, headDepthPoint, ColorImageFormat.RgbResolution640x480Fps30);
//left hand
ColorImagePoint leftColorPoint =
mapper.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, leftDepthPoint,
ColorImageFormat.RgbResolution640x480Fps30);
//right hand
ColorImagePoint rightColorPoint =
mapper.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, rightDepthPoint,
ColorImageFormat.RgbResolution640x480Fps30);
SkeletonPoint rightHand = first.Joints[JointType.HandRight].Position;
SkeletonPoint leftHand = first.Joints[JointType.HandLeft].Position;
if (isInBoundingBox(rightHand))
{
CameraPosition(rightEllipse, rightColorPoint);
RightBlobs.Points.Add(new System.Windows.Point(rightColorPoint.X, rightColorPoint.Y));
//drawBlob(rightColorPoint);
tuioManager.addPoint(0, new PointF(rightColorPoint.X, rightColorPoint.Y));
}
else
{
rightColorPoint.X = 0; rightColorPoint.Y = 0;
CameraPosition(rightEllipse, rightColorPoint);
}
if (isInBoundingBox(leftHand))
{
CameraPosition(leftEllipse, leftColorPoint);
LeftBlobs.Points.Add(new System.Windows.Point(leftColorPoint.X, leftColorPoint.Y));
//drawBlob(leftColorPoint);
tuioManager.addPoint(1, new PointF(leftColorPoint.X, leftColorPoint.Y));
}
else
{
leftColorPoint.X = 0; leftColorPoint.Y = 0;
CameraPosition(leftEllipse, leftColorPoint);
}
//Console.WriteLine("X " + rightHand.X + "Y " + rightHand.Y + "Z " + rightHand.Z);
label1.Content = "X " + rightHand.X + "Y " + rightHand.Y + "Z " + rightHand.Z;
}
}
public List<Tuple<double, double, List<List<Point3D>>>> determineLimb(PointCloud pcdexisting, double weight)
{
//pull in skeleton measures from a temporary file for corbett.parse for now.
kinectInterp = new KinectInterpreter(skeloutline);
Dictionary<String, double[]> jointDepthsStr = new Dictionary<String, double[]>();
//temporary tuple for results
Tuple<double, double, List<List<Point3D>>> T = new Tuple<double, double, List<List<Point3D>>>(0,0,null);
//permanent list of tuples for passing back to coreLoader
List<Tuple<double, double, List<List<Point3D>>>> limbMeasures = new List<Tuple<double,double,List<List<Point3D>>>>();
//Test if we have a kinect otherwise we cannot use coordinate mapper.
if (KinectSensor.KinectSensors.Count > 0)
{
//test if we have already enumerated joint depths, if so, this has followed a recent scan.
if (jointDepths.Count == 0)
{
StreamReader sr = new StreamReader("SKEL.ptemp");
String line;
while ((line = sr.ReadLine()) != null)
{
String[] joint = Regex.Split(line, ":");
String[] positions = Regex.Split(joint[1], ",");
double[] jointPos = { Convert.ToDouble(positions[0]), Convert.ToDouble(positions[1]), Convert.ToDouble(Regex.Split(positions[2], "\n")[0]) };
//convert to depth co-ordinate space
SkeletonPoint sp = new SkeletonPoint();
sp.X = (float)Convert.ToDouble(jointPos[1]);
sp.Y = (float)Convert.ToDouble(jointPos[2]);
sp.Z = (float)Convert.ToDouble(jointPos[0]);
CoordinateMapper cm = new CoordinateMapper(kinectInterp.kinectSensor);
DepthImagePoint dm = cm.MapSkeletonPointToDepthPoint(sp, DepthImageFormat.Resolution640x480Fps30);
//convert x and y co-ords to arbitrary point cloud space
Tuple<double, double, double> convertedPoints = LimbCalculator.convertToPCCoords(dm.X, dm.Y, sp.Z);
double[] jointPos2 = { convertedPoints.Item3, convertedPoints.Item1, convertedPoints.Item2 };
//place back into jointDepths array in terms of depth space.
jointDepthsStr.Add(joint[0], jointPos2);
}
}
else
{
//we have some live skeleton depths, enumerate into strings
foreach(JointType j in jointDepths.Keys) {
jointDepthsStr = new Dictionary<String, double[]>();
jointDepthsStr.Add(j.ToString(),jointDepths[j]);
}
}
for (int limbArea = 1; limbArea <= 8; limbArea++)
{
//pass point cloud and correct bounds to Limb Calculator
//shoulders is first option in list so pass first.
limbMeasures.Add(LimbCalculator.calculateLimbBounds(pcdexisting, jointDepthsStr, limbArea, weight));
}
}
else
{
MessageBoxResult result = System.Windows.MessageBox.Show(this, "You need a Kinect to perform this action.",
"Kinect Sensor Missing", MessageBoxButton.OK, MessageBoxImage.Stop);
}
//change colour of point cloud for limb selection mode
gv.setMaterial();
this.DataContext = gv;
return limbMeasures;
}
/// <summary>
/// Returns the position of a 3D vector on a 2D surface in depth space
/// </summary>
/// <param name="j">current instance</param>
/// <param name="nui">NUI Runtime</param>
/// <param name="panel">The canvas which to project the vector on</param>
/// <returns>2D position</returns>
public static Point GetDisplayPosition(this NUIVector j, KinectSensor nui, Canvas panel)
{
float depthX, depthY;
//nui.SkeletonStream.SkeletonToDepthImage(j, out depthX, out depthY);
CoordinateMapper cm = new CoordinateMapper(nui);
DepthImagePoint dip = cm.MapSkeletonPointToDepthPoint(j, nui.DepthStream.Format);
depthX = dip.X;
depthY = dip.Y;
// crop to panel? - yields 320x240 all the time o.O
//int X = (int)Math.Max(0, Math.Min(depthX * panel.ActualWidth, panel.ActualWidth));
//int Y = (int)Math.Max(0, Math.Min(depthY * panel.ActualHeight, panel.ActualHeight));
return new Point(depthX, depthY);
}
private void DisplayProductOnUser(Skeleton skel, DrawingContext dc)
{
if (skel == null)
{
throw new ArgumentNullException("The skeleton was null upon trying to place product on the user");
}
/* Get the coords of the body */
CoordinateMapper coordMapper = new CoordinateMapper(sensorChooser.Kinect);
DepthImagePoint head = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.Head].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint centerShoulder = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.ShoulderCenter].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightShoulder = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.ShoulderRight].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint leftShoulder = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.ShoulderLeft].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightElbow = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.ElbowRight].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint leftElbow = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.ElbowLeft].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightHip = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.HipRight].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint centerHip = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.HipCenter].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint leftHip = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.HipLeft].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightKnee = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.KneeRight].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint leftKnee = coordMapper.MapSkeletonPointToDepthPoint(skel.Joints[JointType.KneeLeft].Position, DepthImageFormat.Resolution640x480Fps30);
BitmapImage bitImg = null;
if (File.Exists(Product.ProductImage + ".png"))
{
bitImg = new BitmapImage(new Uri(Product.ProductImage + ".png"));
}
else
{
bitImg = new BitmapImage(new Uri(System.AppDomain.CurrentDomain.BaseDirectory + "Resources\\ProductImages\\tempImage.png"));
}
ImageSource image = bitImg;
DrawingGroup drawingGroup = new DrawingGroup();
double angleRotate = GetAngle(centerHip, centerShoulder);
double imageHeight = image.Height;
double imageWidth = image.Width;
double ratio = imageHeight / imageWidth;
// create variables
Point imagePoint = new Point();
double x, y, xcoord, ycoord, width;
if (Product.ParentSub.ToLower() == "jeans" || Product.ParentSub.ToLower() == "shorts" || Product.ParentSub.ToLower() == "skirts" ||
(Product.ParentCat.ToLower() == "mens" && Product.ParentSub.ToLower() == "suits" && Product.Title.ToLower().Contains("trouser")))
{
centerHip.Y += 50;
if (Product.ParentSub.ToLower() == "skirts" &&
Product.IsThin)
{
width = GetShoulderWidth(skel) * 1.5;
}
else
{
width = GetShoulderWidth(skel) * 1.2;
}
imagePoint = GetXYCoordToUse(centerHip, width, angleRotate);
x = imagePoint.X;
y = imagePoint.Y;
if (Math.Abs(leftHip.Depth - rightHip.Depth) < 10)
{
ClothingHeight = ratio * width;
}
xcoord = centerHip.X;
ycoord = centerHip.Y;
}
else
{
centerShoulder.Y += 10;
if (Product.IsThin)
{
width = GetShoulderWidth(skel);
}
else
{
width = GetShoulderWidth(skel) * 1.5;
}
imagePoint = GetXYCoordToUse(centerShoulder, width, angleRotate);
x = imagePoint.X;
y = imagePoint.Y;
if (Math.Abs(leftShoulder.Depth - rightShoulder.Depth) < 10)
{
ClothingHeight = ratio * width;
}
xcoord = centerShoulder.X;
ycoord = centerShoulder.Y;
}
if (angleRotate > 0)
{
x = x + (angleRotate * 1.15);
}
else if (angleRotate < 0)
{
x = x - (Math.Abs(angleRotate) * 1.15);
}
if (ClothingHeight != 0)
{
drawingGroup.Transform = new RotateTransform(angleRotate, xcoord, ycoord - 50);
drawingGroup.Children.Add(new ImageDrawing(image, new Rect(x, y, width, ClothingHeight)));
dc.DrawDrawing(drawingGroup);
}
else
{
dc.DrawText(new FormattedText("Please stand straight for configuration", CultureInfo.GetCultureInfo("en-us"), FlowDirection.LeftToRight, new Typeface("Segoe"), 24, Brushes.White), new Point(centerShoulder.X - 200, centerShoulder.Y + 100));
}
}
public static Vector2 Convert(CoordinateMapper mapper, SkeletonPoint position,
Object format = null)
{
float width = 0;
float height = 0;
float x = 0;
float y = 0;
if (format != null)
{
if (format is ColorImageFormat)
{
var colorFormat = (ColorImageFormat)format;
var colorPoint = mapper.MapSkeletonPointToColorPoint(position, colorFormat);
x = colorPoint.X;
y = colorPoint.Y;
switch (colorFormat)
{
case ColorImageFormat.RawYuvResolution640x480Fps15:
case ColorImageFormat.RgbResolution640x480Fps30:
case ColorImageFormat.YuvResolution640x480Fps15:
width = 640;
height = 480;
break;
case ColorImageFormat.RgbResolution1280x960Fps12:
width = 1280;
height = 960;
break;
}
}
else if (format is DepthImageFormat)
{
var depthFormat = (DepthImageFormat)format;
var depthPoint = mapper.MapSkeletonPointToDepthPoint(position, depthFormat);
x = depthPoint.X;
y = depthPoint.Y;
switch (depthFormat)
{
case DepthImageFormat.Resolution80x60Fps30:
width = 80;
height = 60;
break;
case DepthImageFormat.Resolution320x240Fps30:
width = 320;
height = 240;
break;
case DepthImageFormat.Resolution640x480Fps30:
width = 640;
height = 480;
break;
}
}
else
{
width = 1;
height = 1;
}
}
return(new Vector2(x / width, y / height));
}
private void KinectSensorOnAllFramesReady(object sender, AllFramesReadyEventArgs e)
{
if (frameProccessed[1] == false)
{
frameProccessed[1] = true;
}
else
{
frameProccessed[1] = false;
return;
}
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = e.OpenColorImageFrame();
depthImageFrame = e.OpenDepthImageFrame();
skeletonFrame = e.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
using (depthImageFrame)
{
if (depthImageFrame != null && skeletonData != null)
{
foreach (Skeleton sd in skeletonData)
{
if (sd.TrackingState == SkeletonTrackingState.Tracked || sd.TrackingState == SkeletonTrackingState.PositionOnly)
{
Joint joint = sd.Joints[JointType.Head];
DepthImagePoint depthPoint;
CoordinateMapper coordinateMapper = new CoordinateMapper(frontSensor);
depthPoint = coordinateMapper.MapSkeletonPointToDepthPoint(joint.Position, DepthImageFormat.Resolution320x240Fps30);
point = new System.Windows.Point((int)(frontSensor.ColorStream.FrameWidth * depthPoint.X
/ depthImageFrame.Width),
(int)(frontSensor.ColorStream.FrameHeight * depthPoint.Y
/ depthImageFrame.Height));
/* textBlock1.Text = string.Format("X:{0:0.00} Y:{1:0.00} Z:{2:0.00}",
point.X,
point.Y,
joint.Position.Z); */
Canvas.SetLeft(headEllipse, point.X - headEllipse.Width / 2);
Canvas.SetTop(headEllipse, point.Y - headEllipse.Height / 2);
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(frontSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTrack
// is unable to be instantiated. Catch that exception
// and don't track a face.
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, sd);
if (frame.TrackSuccessful)
{
faceTriangles = frame.GetTriangles();
this.facePoints = frame.GetProjected3DShape();
var faceModelPts = new List<Point>();
var faceModel = new List<FaceModelTriangle>();
for (int i = 0; i < this.facePoints.Count; i++)
{
faceModelPts.Add(new Point(this.facePoints[i].X + 0.5f, this.facePoints[i].Y + 0.5f));
}
foreach (var t in faceTriangles)
{
var triangle = new FaceModelTriangle();
triangle.P1 = faceModelPts[t.First];
//triangle.P2 = faceModelPts[t.Second];
//triangle.P3 = faceModelPts[t.Third];
faceModel.Add(triangle);
}
Canvas.SetLeft(noseEllipse, faceModel[108].P1.X - noseEllipse.Width / 2);
Canvas.SetTop(noseEllipse, faceModel[108].P1.Y - noseEllipse.Height / 2);
nosePoint = new Point(faceModel[108].P1.X, faceModel[108].P1.Y);
}
}
}
}
}
}
getAttentionAngle(nosePoint);
}
}
public List <Tuple <double, double, List <List <Point3D> > > > determineLimb(PointCloud pcdexisting, double weight)
{
//pull in skeleton measures from a temporary file for corbett.parse for now.
kinectInterp = new KinectInterpreter(skeloutline);
Dictionary <String, double[]> jointDepthsStr = new Dictionary <String, double[]>();
//temporary tuple for results
Tuple <double, double, List <List <Point3D> > > T = new Tuple <double, double, List <List <Point3D> > >(0, 0, null);
//permanent list of tuples for passing back to coreLoader
List <Tuple <double, double, List <List <Point3D> > > > limbMeasures = new List <Tuple <double, double, List <List <Point3D> > > >();
//Test if we have a kinect otherwise we cannot use coordinate mapper.
if (KinectSensor.KinectSensors.Count > 0)
{
//test if we have already enumerated joint depths, if so, this has followed a recent scan.
if (jointDepths.Count == 0)
{
StreamReader sr = new StreamReader("SKEL.ptemp");
String line;
while ((line = sr.ReadLine()) != null)
{
String[] joint = Regex.Split(line, ":");
String[] positions = Regex.Split(joint[1], ",");
double[] jointPos = { Convert.ToDouble(positions[0]), Convert.ToDouble(positions[1]), Convert.ToDouble(Regex.Split(positions[2], "\n")[0]) };
//convert to depth co-ordinate space
SkeletonPoint sp = new SkeletonPoint();
sp.X = (float)Convert.ToDouble(jointPos[1]);
sp.Y = (float)Convert.ToDouble(jointPos[2]);
sp.Z = (float)Convert.ToDouble(jointPos[0]);
CoordinateMapper cm = new CoordinateMapper(kinectInterp.kinectSensor);
DepthImagePoint dm = cm.MapSkeletonPointToDepthPoint(sp, DepthImageFormat.Resolution640x480Fps30);
//convert x and y co-ords to arbitrary point cloud space
Tuple <double, double, double> convertedPoints = LimbCalculator.convertToPCCoords(dm.X, dm.Y, sp.Z);
double[] jointPos2 = { convertedPoints.Item3, convertedPoints.Item1, convertedPoints.Item2 };
//place back into jointDepths array in terms of depth space.
jointDepthsStr.Add(joint[0], jointPos2);
}
}
else
{
//we have some live skeleton depths, enumerate into strings
foreach (JointType j in jointDepths.Keys)
{
jointDepthsStr = new Dictionary <String, double[]>();
jointDepthsStr.Add(j.ToString(), jointDepths[j]);
}
}
for (int limbArea = 1; limbArea <= 8; limbArea++)
{
//pass point cloud and correct bounds to Limb Calculator
//shoulders is first option in list so pass first.
limbMeasures.Add(LimbCalculator.calculateLimbBounds(pcdexisting, jointDepthsStr, limbArea, weight));
}
}
else
{
MessageBoxResult result = System.Windows.MessageBox.Show(this, "You need a Kinect to perform this action.",
"Kinect Sensor Missing", MessageBoxButton.OK, MessageBoxImage.Stop);
}
//change colour of point cloud for limb selection mode
gv.setMaterial();
this.DataContext = gv;
return(limbMeasures);
}
/// <summary>
/// Gets called if the RGB, depth and skeleton frame is ready
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void mainKinectSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
if (!tutorial) return;
Skeleton mainSkeleton = getFirstSkeleton(e); //get the first skeleton
if (mainSkeleton == null) return; //return if the Kinect does not recognize any skeletons
CoordinateMapper mapper = new CoordinateMapper(mainKinectSensor); //mapper between skeleton and depth image
if (!corrected)
{
DepthImagePoint pointHead = mapper.MapSkeletonPointToDepthPoint(mainSkeleton.Joints[JointType.Head].Position, DepthImageFormat.Resolution640x480Fps30);
if (pointHead.Y < 100)
{
mainKinectSensor.ElevationAngle += Convert.ToInt32((120 - pointHead.Y) / TILT_FACTOR);
corrected = true;
return;
}
if (pointHead.Y > 140)
{
mainKinectSensor.ElevationAngle -= Convert.ToInt32((pointHead.Y - 120) / TILT_FACTOR);
corrected = true;
return;
}
}
corrected = true;
DepthImagePoint pointRight = mapper.MapSkeletonPointToDepthPoint(mainSkeleton.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30); //get the right hand
DepthImagePoint pointLeft = mapper.MapSkeletonPointToDepthPoint(mainSkeleton.Joints[JointType.HandLeft].Position, DepthImageFormat.Resolution640x480Fps30); //get the left hand
if (rotation & Math.Abs(pointLeft.X - pointRight.X) < HANDS_DISTANCE)
{
mainWindow.setHandMarkers(pointRight.X, pointRight.Y, pointLeft.X, pointLeft.Y, true);
if (!playing) return;
if (oldAngle == -1)
{
oldAngle = pointLeft.Y - pointRight.Y;
}
room.rotateCurrentElement(((pointLeft.Y - pointRight.Y) - oldAngle) / 2);
oldAngle = pointLeft.Y - pointRight.Y;
}
else
{
oldAngle = -1;
mainWindow.setHandMarkers(pointRight.X, pointRight.Y, pointLeft.X, pointLeft.Y, false);
}
if (!playing) return;
if (absolute)
{
if (zMax == -1 && zMin == -1)
{
zMax = pointRight.Depth + 400;
zMin = pointRight.Depth - 400;
}
if (pointRight.Depth > zMax) //adapt the Z-Range if the player goes out of it
{
zMax = pointRight.Depth;
zMin = pointRight.Depth - 800;
}
if (pointRight.Depth < zMin)
{
zMax = pointRight.Depth + 800;
zMin = pointRight.Depth;
}
Console.WriteLine(pointRight.Depth - zMin);
room.translateCurrentElementAbsolute(pointRight.X, pointRight.Y, pointRight.Depth - zMin); //absolute movement
}
else
{
if (oldx == -1) //relative movement
{
oldx = pointRight.X;
}
if (oldy == -1)
{
oldy = pointRight.Y;
}
if (oldz == -1)
{
oldz = pointRight.Depth;
}
room.translateCurrentElementRelative((pointRight.X - oldx) * Room.FACTOR_X, 0, 0);
room.translateCurrentElementRelative(0, (oldy - pointRight.Y) * Room.FACTOR_Y, 0);
room.translateCurrentElementRelative(0, 0, (pointRight.Depth - oldz) * Room.FACTOR_Z);
oldx = pointRight.X;
oldy = pointRight.Y;
oldz = pointRight.Depth;
}
}
/// <summary>
/// Depth Frame Ready
/// Kinect tracks skeletons, localize the Skeleton Hand Joint Position
/// Map Skeleton point given by Kinect to PointDepth3D , Add result to HandTracker's List buffer for computing every frame
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
void sensor_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
{
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame != null)
{
if (skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
skeletonFrame.CopySkeletonDataTo(skeletonData);
int personCount = 0;
foreach (Skeleton sk in skeletonData)
{
if (sk.TrackingState == SkeletonTrackingState.Tracked)
{
//Skeleton point map to depth point given by Kinect
CoordinateMapper mapper = new CoordinateMapper(this.sensor);
DepthImagePoint left = mapper.MapSkeletonPointToDepthPoint(
sk.Joints[JointType.HandLeft].Position,
this.sensor.DepthStream.Format);
DepthImagePoint right = mapper.MapSkeletonPointToDepthPoint(
sk.Joints[JointType.HandRight].Position,
this.sensor.DepthStream.Format);
rightWrist = new PointSkeleton3D(sk.Joints[JointType.WristRight].Position);
rightHand = new PointSkeleton3D(sk.Joints[JointType.HandRight].Position);
rightElbow = new PointSkeleton3D(sk.Joints[JointType.ElbowRight].Position);
personCount++;
}
}
skeletonReadyFlag = (personCount == 0 ? false : true);
}
}
}
/*
//output real image
void nui_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
{
ColorImageFrame ImageParam = e.OpenColorImageFrame();
if (ImageParam == null) return;
byte[] ImageBits = new byte[ImageParam.PixelDataLength];
ImageParam.CopyPixelDataTo(ImageBits);
BitmapSource src = null;
src = BitmapSource.Create(ImageParam.Width, //create bitmap source
ImageParam.Height,
96, 96,
PixelFormats.Bgr32, //Bgra32 pixel format, r g, b, a that each 8 bits
null,
ImageBits, // real image data
ImageParam.Width * ImageParam.BytesPerPixel);
image1.Source = src; //Stopped image
}
*/
void nui_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
if (!((KinectSensor)sender).SkeletonStream.IsEnabled) { return; }
SkeletonFrame sf = e.OpenSkeletonFrame();
if (sf == null) return;
Skeleton[] skeltonData = new Skeleton[sf.SkeletonArrayLength];
//System.Diagnostics.Debug.WriteLine("SkeletonArrayLength: {0}", sf.SkeletonArrayLength);
sf.CopySkeletonDataTo(skeltonData);
using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame())
{
if (depthImageFrame != null)
{
foreach (Skeleton sd in skeltonData) //inhence statement in JAVA
{
if (sd.TrackingState == SkeletonTrackingState.Tracked)
{
Joint joint = sd.Joints[JointType.HandRight];
/*Coordinaate transform*/
CoordinateMapper coordMapper = new CoordinateMapper(nui);
DepthImagePoint depthPoint = coordMapper.MapSkeletonPointToDepthPoint(joint.Position, depthImageFrame.Format);
Point point = new Point((int)(image1.Width * depthPoint.X / depthImageFrame.Width),
(int)(image1.Height * depthPoint.Y / depthImageFrame.Height));
Canvas.SetLeft(Image6, point.X);
Canvas.SetTop(Image6, point.Y);
/**********************MENU 1********************/
if (menu_flag1)
{
System.Diagnostics.Debug.WriteLine("Inside Menu 1");
//////////////////Button1 : game start after 3 seconds if wait on the button
if ((point.X > 100 && point.X < 500) && (point.Y > 100 && point.Y < 200) && menu_flag1)
{
System.Diagnostics.Debug.WriteLine("Inside game start Button1");
//game_start(timer_flag);
if (timer_flag)
{
myTimer.Tick += new EventHandler(myTimer_Tick_Start);//call the myTimer_Tick_Start function
myTimer.Stop();
timer_flag = false;
myTimer.Interval = 3000;
myTimer.Start();
}
}
//////////////////Button2 : quit game after 3 seconds if wait on the button
else if ((point.X > 100 && point.X < 500) && (point.Y > 300 && point.Y < 400) && menu_flag1)
{
System.Diagnostics.Debug.WriteLine("Inside quit game Button2");
if (timer_flag)
{
myTimer.Tick += new EventHandler(myTimer_Tick_Finish);//call the myTimer_Tick_Finish Function
myTimer.Stop();
timer_flag = false;
myTimer.Interval = 3000;
myTimer.Start();
}
}
}
/****************************MENU 2*************************************/
if (menu_flag2)
{
System.Diagnostics.Debug.WriteLine("Inside Menu 2");
//////////////////Button1 : Create game after 3 seconds if wait on the button
if ((point.X > 75 && point.X < 200) && (point.Y > 100 && point.Y < 250) && menu_flag2)
{
System.Diagnostics.Debug.WriteLine("Inside create Button1");
if (timer_flag)
{
myTimer.Tick += new EventHandler(myTimer_Tick_Create_Room);//call the myTimer_Tick_Create_Room Function
myTimer.Stop();
timer_flag = false;
myTimer.Interval = 3000;
myTimer.Start();
}
}
//////////////////Button2 : Game random matching after 3 seconds if wait on the button
else if ((point.X > 237.5 && point.X < 362.5) && (point.Y > 100 && point.Y < 250) && menu_flag2)
{
System.Diagnostics.Debug.WriteLine("Inside game Matching Button2");
if (timer_flag)
{
myTimer.Tick += new EventHandler(myTimer_Tick_Matching);//call the myTimer_Tick_Matching function
myTimer.Stop();
timer_flag = false;
myTimer.Interval = 3000;
myTimer.Start();
}
}
///////////////////Button3 : Back to MENU 1 after 3 seconds if wait on the button
else if ((point.X > 400 && point.X < 525) && (point.Y > 100 && point.Y < 250) && menu_flag2)
{
System.Diagnostics.Debug.WriteLine("Inside back to Menu Button3");
if (timer_flag)
{
myTimer.Tick += new EventHandler(myTimer_Tick_Return_Menu1);//call the myTimer_Tick_Return_Menu1 function
myTimer.Stop();
timer_flag = false;
myTimer.Interval = 3000;
myTimer.Start();
}
}
else
{
myTimer.Stop();
myTimer.Enabled = true;
timer_flag = true;
}
}
}
}
}
}
}
public DepthImagePoint MapSkeletonPointToDepthPoint(SkeletonPoint sp)
{
return(mapper.MapSkeletonPointToDepthPoint(sp, dif));
}
private void KinectSensorOnAllFramesReady(object sender, AllFramesReadyEventArgs e)
{
if (frameProccessed[1] == false)
{
frameProccessed[1] = true;
}
else
{
frameProccessed[1] = false;
return;
}
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = e.OpenColorImageFrame();
depthImageFrame = e.OpenDepthImageFrame();
skeletonFrame = e.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
using (depthImageFrame)
{
if (depthImageFrame != null && skeletonData != null)
{
foreach (Skeleton sd in skeletonData)
{
if (sd.TrackingState == SkeletonTrackingState.Tracked || sd.TrackingState == SkeletonTrackingState.PositionOnly)
{
Joint joint = sd.Joints[JointType.Head];
DepthImagePoint depthPoint;
CoordinateMapper coordinateMapper = new CoordinateMapper(frontSensor);
depthPoint = coordinateMapper.MapSkeletonPointToDepthPoint(joint.Position, DepthImageFormat.Resolution320x240Fps30);
point = new System.Windows.Point((int)(frontSensor.ColorStream.FrameWidth * depthPoint.X
/ depthImageFrame.Width),
(int)(frontSensor.ColorStream.FrameHeight * depthPoint.Y
/ depthImageFrame.Height));
/* textBlock1.Text = string.Format("X:{0:0.00} Y:{1:0.00} Z:{2:0.00}",
* point.X,
* point.Y,
* joint.Position.Z); */
Canvas.SetLeft(headEllipse, point.X - headEllipse.Width / 2);
Canvas.SetTop(headEllipse, point.Y - headEllipse.Height / 2);
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(frontSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTrack
// is unable to be instantiated. Catch that exception
// and don't track a face.
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, sd);
if (frame.TrackSuccessful)
{
faceTriangles = frame.GetTriangles();
this.facePoints = frame.GetProjected3DShape();
var faceModelPts = new List <Point>();
var faceModel = new List <FaceModelTriangle>();
for (int i = 0; i < this.facePoints.Count; i++)
{
faceModelPts.Add(new Point(this.facePoints[i].X + 0.5f, this.facePoints[i].Y + 0.5f));
}
foreach (var t in faceTriangles)
{
var triangle = new FaceModelTriangle();
triangle.P1 = faceModelPts[t.First];
//triangle.P2 = faceModelPts[t.Second];
//triangle.P3 = faceModelPts[t.Third];
faceModel.Add(triangle);
}
Canvas.SetLeft(noseEllipse, faceModel[108].P1.X - noseEllipse.Width / 2);
Canvas.SetTop(noseEllipse, faceModel[108].P1.Y - noseEllipse.Height / 2);
nosePoint = new Point(faceModel[108].P1.X, faceModel[108].P1.Y);
}
}
}
}
}
}
getAttentionAngle(nosePoint);
}
}
internal void ProcessFrame(CoordinateMapper mapper, Skeleton skeletonOfInterest, DepthImageFormat depthImageFormat)
{
_joints.Clear();
if (skeletonOfInterest != null)
{
var size = FormatHelper.GetDepthSize(depthImageFormat);
var depthWidth = (int)size.Width;
var headJoint = skeletonOfInterest.Joints[JointType.Head];
var neckJoint = skeletonOfInterest.Joints[JointType.ShoulderCenter];
var _headPoint = mapper.MapSkeletonPointToDepthPoint(headJoint.Position, depthImageFormat);
var _neckPoint = mapper.MapSkeletonPointToDepthPoint(neckJoint.Position, depthImageFormat);
_headPoint.X = depthWidth - _headPoint.X;
_neckPoint.X = depthWidth - _neckPoint.X;
_joints.Add(_headPoint);
_joints.Add(_neckPoint);
}
RaiseFrameUpdated();
}
/// <summary>
/// Gets called if the RGB, depth and skeleton frame is ready
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void mainKinectSensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
if (!tutorial)
{
return;
}
Skeleton mainSkeleton = getFirstSkeleton(e); //get the first skeleton
if (mainSkeleton == null)
{
return; //return if the Kinect does not recognize any skeletons
}
CoordinateMapper mapper = new CoordinateMapper(mainKinectSensor); //mapper between skeleton and depth image
if (!corrected)
{
DepthImagePoint pointHead = mapper.MapSkeletonPointToDepthPoint(mainSkeleton.Joints[JointType.Head].Position, DepthImageFormat.Resolution640x480Fps30);
if (pointHead.Y < 100)
{
mainKinectSensor.ElevationAngle += Convert.ToInt32((120 - pointHead.Y) / TILT_FACTOR);
corrected = true;
return;
}
if (pointHead.Y > 140)
{
mainKinectSensor.ElevationAngle -= Convert.ToInt32((pointHead.Y - 120) / TILT_FACTOR);
corrected = true;
return;
}
}
corrected = true;
DepthImagePoint pointRight = mapper.MapSkeletonPointToDepthPoint(mainSkeleton.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30); //get the right hand
DepthImagePoint pointLeft = mapper.MapSkeletonPointToDepthPoint(mainSkeleton.Joints[JointType.HandLeft].Position, DepthImageFormat.Resolution640x480Fps30); //get the left hand
if (rotation & Math.Abs(pointLeft.X - pointRight.X) < HANDS_DISTANCE)
{
mainWindow.setHandMarkers(pointRight.X, pointRight.Y, pointLeft.X, pointLeft.Y, true);
if (!playing)
{
return;
}
if (oldAngle == -1)
{
oldAngle = pointLeft.Y - pointRight.Y;
}
room.rotateCurrentElement(((pointLeft.Y - pointRight.Y) - oldAngle) / 2);
oldAngle = pointLeft.Y - pointRight.Y;
}
else
{
oldAngle = -1;
mainWindow.setHandMarkers(pointRight.X, pointRight.Y, pointLeft.X, pointLeft.Y, false);
}
if (!playing)
{
return;
}
if (absolute)
{
if (zMax == -1 && zMin == -1)
{
zMax = pointRight.Depth + 400;
zMin = pointRight.Depth - 400;
}
if (pointRight.Depth > zMax) //adapt the Z-Range if the player goes out of it
{
zMax = pointRight.Depth;
zMin = pointRight.Depth - 800;
}
if (pointRight.Depth < zMin)
{
zMax = pointRight.Depth + 800;
zMin = pointRight.Depth;
}
Console.WriteLine(pointRight.Depth - zMin);
room.translateCurrentElementAbsolute(pointRight.X, pointRight.Y, pointRight.Depth - zMin); //absolute movement
}
else
{
if (oldx == -1) //relative movement
{
oldx = pointRight.X;
}
if (oldy == -1)
{
oldy = pointRight.Y;
}
if (oldz == -1)
{
oldz = pointRight.Depth;
}
room.translateCurrentElementRelative((pointRight.X - oldx) * Room.FACTOR_X, 0, 0);
room.translateCurrentElementRelative(0, (oldy - pointRight.Y) * Room.FACTOR_Y, 0);
room.translateCurrentElementRelative(0, 0, (pointRight.Depth - oldz) * Room.FACTOR_Z);
oldx = pointRight.X;
oldy = pointRight.Y;
oldz = pointRight.Depth;
}
}
//DEPTH_END
void GetCameraPoint(Skeleton first, AllFramesReadyEventArgs e)
{
using (DepthImageFrame depth = e.OpenDepthImageFrame())
{
if (depth == null || kinectSensorChooser1.Kinect == null)
{
return;
}
CoordinateMapper cm = new CoordinateMapper(kinectSensorChooser1.Kinect);
DepthImagePoint headDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.Head].Position,DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint leftDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandLeft].Position, DepthImageFormat.Resolution640x480Fps30);
DepthImagePoint rightDepthPoint = cm.MapSkeletonPointToDepthPoint(first.Joints[JointType.HandRight].Position, DepthImageFormat.Resolution640x480Fps30);
ColorImagePoint headColorPoint = cm.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, headDepthPoint, ColorImageFormat.RgbResolution640x480Fps30);
ColorImagePoint leftColorPoint = cm.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, leftDepthPoint, ColorImageFormat.RgbResolution640x480Fps30);
ColorImagePoint rightColorPoint = cm.MapDepthPointToColorPoint(DepthImageFormat.Resolution640x480Fps30, rightDepthPoint, ColorImageFormat.RgbResolution640x480Fps30);
//=> CameraPosition(headImage, headColorPoint);
//=> CameraPosition(leftEllipse, leftColorPoint);
//=> CameraPosition(rightEllipse, rightColorPoint);
}
}
