private KinectHelper(TransformSmoothParameters tsp, bool near = false,
ColorImageFormat colorFormat = ColorImageFormat.RgbResolution1280x960Fps12,
DepthImageFormat depthFormat = DepthImageFormat.Resolution640x480Fps30)
{
_kinectSensor = KinectSensor.KinectSensors.FirstOrDefault(s => s.Status == KinectStatus.Connected);
if (_kinectSensor == null)
{
throw new Exception("No Kinect-Sensor found.");
}
if (near)
{
_kinectSensor.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated;
_kinectSensor.DepthStream.Range = DepthRange.Near;
_kinectSensor.SkeletonStream.EnableTrackingInNearRange = true;
}
DepthImageFormat = depthFormat;
ColorImageFormat = colorFormat;
_kinectSensor.SkeletonStream.Enable(tsp);
_kinectSensor.ColorStream.Enable(colorFormat);
_kinectSensor.DepthStream.Enable(depthFormat);
_kinectSensor.AllFramesReady += AllFramesReady;
_kinectSensor.Start();
_faceTracker = new FaceTracker(_kinectSensor);
}
public void ProcessData(KinectSensor kinectSensor, ColorImageFormat colorImageFormat,
byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage,
Skeleton[] skeletons, int skeletonFrameNumber)
{
if (skeletons == null)
{
return;
}
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in skeletons)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this._trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this._trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this._trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(kinectSensor, colorImageFormat, colorImage, depthImageFormat,
depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrameNumber;
}
}
}
RemoveOldTrackers(skeletonFrameNumber);
}
/*/////////////////////////////////////////
* CONSTRUCTOR(S)/DESTRUCTOR(S)
*/
///////////////////////////////////////
public KinectManager(ColorImageFormat p_colour_format,
DepthImageFormat p_depth_format,
KinectGame_WindowsXNA p_game)
{
// Initialise the Kinect selector...
this.colour_image_format = p_colour_format;
this.depth_image_format = p_depth_format;
this.root_game = p_game;
this.colour_stream = null;
this.depth_stream = null;
this.skeleton_stream = null;
this.debug_video_stream_dimensions = new Vector2(200, 150);
status_map = new Dictionary<KinectStatus, string>();
KinectSensor.KinectSensors.StatusChanged += this.KinectSensorsStatusChanged; // handler function for changes in the Kinect system
this.DiscoverSensor();
this.status_map.Add(KinectStatus.Undefined, "UNKNOWN STATUS MESSAGE");
this.status_map.Add(KinectStatus.Connected, "Connected.");//string.Empty);
this.status_map.Add(KinectStatus.DeviceNotGenuine, "Detected device is not genuine!");
this.status_map.Add(KinectStatus.DeviceNotSupported, "Detected device is not supported!");
this.status_map.Add(KinectStatus.Disconnected, "Disconnected/Device required!");
this.status_map.Add(KinectStatus.Error, "Error in Kinect sensor!");
this.status_map.Add(KinectStatus.Initializing, "Initialising Kinect sensor...");
this.status_map.Add(KinectStatus.InsufficientBandwidth, "Insufficient bandwidth for Kinect sensor!");
this.status_map.Add(KinectStatus.NotPowered, "Detected device is not powered!");
this.status_map.Add(KinectStatus.NotReady, "Detected device is not ready!");
// Load the status message font:
this.msg_font = this.root_game.Content.Load<SpriteFont>("Fonts/Segoe16");
this.msg_label_pos = new Vector2(4.0f, 2.0f);
}
public VideoShot(ColorImageProcesser processer, MainWindow window, int videoNum,
KinectSensor kinectDevice,
int dWidht, int dHeight,
int cWidth, int cHeight,
DepthImageFormat dImageFormat, ColorImageFormat cImageFormat)
{
parentProcesser = processer;
videoName = PadLeft(videoNum);
_windowUI = window;
_kinectDevice = kinectDevice;
depthFrameWidth = dWidht;
depthFrameHeight = dHeight;
colorFrameWidth = cWidth;
colorFrameHeight = cHeight;
depthFrameStride = depthFrameWidth * BytesPerPixel;
colorFrameStride = colorFrameWidth * BytesPerPixel;
depthImageFormat = dImageFormat;
colorImageFormat = cImageFormat;
screenHeight = SystemParameters.PrimaryScreenHeight;
screenWidth = SystemParameters.PrimaryScreenWidth;
Start();
}
public CoordinateConverter(IEnumerable<byte> kinectParams, ColorImageFormat cif,
DepthImageFormat dif)
{
mapper = new CoordinateMapper(kinectParams);
this.cif = cif;
this.dif = dif;
}
/// <summary>
/// コンストラクタ
/// </summary>
public MainWindow()
{
InitializeComponent();
// Init Kinect Sensors
this.kinect = KinectSensor.GetDefault();
if (kinect == null)
{
this.showCloseDialog("Kinectが接続されていないか、利用できません。アプリケーションを終了します。");
}
this.colorImageFormat = ColorImageFormat.Bgra;
this.colorFrameDescription = this.kinect.ColorFrameSource.CreateFrameDescription(this.colorImageFormat);
this.colorFrameReader = this.kinect.ColorFrameSource.OpenReader();
this.colorFrameReader.FrameArrived += ColorFrameReader_FrameArrived;
bodyFrameReader = kinect.BodyFrameSource.OpenReader();
bodyFrameReader.FrameArrived += bodyFrameReader_FrameArrived;
this.kinect.Open();
this.bodies = this.bodies = new Body[kinect.BodyFrameSource.BodyCount];
KinectRegion.SetKinectRegion(this, kinectRegion);
this.kinectRegion.KinectSensor = KinectSensor.GetDefault();
this.isTraining = false;
}
public SkeletalTracker(Rectangle fullscreen, CoordinateMapper coordinateMapper, ColorImageFormat colorFormat)
{
_coordinateMapper = coordinateMapper;
_colorFormat = colorFormat;
_fullscreen = fullscreen;
_aspectRatio = _fullscreen.Width / (double)_fullscreen.Height;
}
/// <summary>
/// Initializes a new instance of the FaceTracker class from a reference of the Kinect device.
/// <param name="sensor">Reference to kinect sensor instance</param>
/// </summary>
public FaceTracker(KinectSensor sensor)
{
if (sensor == null) {
throw new ArgumentNullException("sensor");
}
if (!sensor.ColorStream.IsEnabled) {
throw new InvalidOperationException("Color stream is not enabled yet.");
}
if (!sensor.DepthStream.IsEnabled) {
throw new InvalidOperationException("Depth stream is not enabled yet.");
}
this.operationMode = OperationMode.Kinect;
this.coordinateMapper = sensor.CoordinateMapper;
this.initializationColorImageFormat = sensor.ColorStream.Format;
this.initializationDepthImageFormat = sensor.DepthStream.Format;
var newColorCameraConfig = new CameraConfig(
(uint)sensor.ColorStream.FrameWidth,
(uint)sensor.ColorStream.FrameHeight,
sensor.ColorStream.NominalFocalLengthInPixels,
FaceTrackingImageFormat.FTIMAGEFORMAT_UINT8_B8G8R8X8);
var newDepthCameraConfig = new CameraConfig(
(uint)sensor.DepthStream.FrameWidth,
(uint)sensor.DepthStream.FrameHeight,
sensor.DepthStream.NominalFocalLengthInPixels,
FaceTrackingImageFormat.FTIMAGEFORMAT_UINT16_D13P3);
this.Initialize(newColorCameraConfig, newDepthCameraConfig, IntPtr.Zero, IntPtr.Zero, this.DepthToColorCallback);
}
private void KinectSensorOnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
using (var colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame())
{
if (colorImageFrame == null)
{
return;
}
// Make a copy of the color frame for displaying.
var haveNewFormat = this.currentColorImageFormat != colorImageFrame.Format;
if (haveNewFormat)
{
this.currentColorImageFormat = colorImageFrame.Format;
this.colorImageData = new byte[colorImageFrame.PixelDataLength];
this.colorImageWritableBitmap = new WriteableBitmap(
colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null);
ColorImage.Source = this.colorImageWritableBitmap;
}
colorImageFrame.CopyPixelDataTo(this.colorImageData);
this.colorImageWritableBitmap.WritePixels(
new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height),
this.colorImageData,
colorImageFrame.Width * Bgr32BytesPerPixel,
0);
}
}
public MainWindow()
{
InitializeComponent();
try
{
///キネクト本体の接続を確保、たしか接続されてない場合はfalseとかになった記憶
this.kinect = KinectSensor.GetDefault();
///読み込む画像のフォーマット(rgbとか)を指定、どうやって読み込むかのリーダの設定も
this.colorImageFormat = ColorImageFormat.Bgra;
this.colorFrameDescription = this.kinect.ColorFrameSource.CreateFrameDescription(this.colorImageFormat);
this.colorFrameReader = this.kinect.ColorFrameSource.OpenReader();
this.colorFrameReader.FrameArrived += colorFrameReader_FrameArrived;
this.kinect.Open();//キネクト起動！！
if (!kinect.IsOpen)
{
this.errorLog.Visibility = Visibility.Visible;
this.errorLog.Content = "キネクトが見つからないよ！残念！";
throw new Exception("キネクトが見つかりませんでした！！！");
}
///bodyを格納するための配列作成
bodies = new Body[kinect.BodyFrameSource.BodyCount];
///ボディリーダーを開く
bodyFrameReader = kinect.BodyFrameSource.OpenReader();
bodyFrameReader.FrameArrived += bodyFrameReader_FrameArrived;
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
Close();
}
}
/// <summary>
/// Initializes a new instance of the KinectChooser class.
/// </summary>
/// <param name="game">The related game object.</param>
/// <param name="colorFormat">The desired color image format.</param>
/// <param name="depthFormat">The desired depth image format.</param>
public KinectChooser(Game game, ColorImageFormat colorFormat, DepthImageFormat depthFormat)
: base(game)
{
this.colorImageFormat = colorFormat;
this.depthImageFormat = depthFormat;
this.nearMode = false;
this.seatedMode = false;
this.SimulateMouse = false;
if (!Game1.SIMULATE_NO_KINECT)
{
KinectSensor.KinectSensors.StatusChanged += this.KinectSensors_StatusChanged;
this.DiscoverSensor();
}
this.statusMap.Add(KinectStatus.Undefined, "Not connected, or in use");
this.statusMap.Add(KinectStatus.Connected, string.Empty);
this.statusMap.Add(KinectStatus.DeviceNotGenuine, "Device Not Genuine");
this.statusMap.Add(KinectStatus.DeviceNotSupported, "Device Not Supported");
this.statusMap.Add(KinectStatus.Disconnected, "Required");
this.statusMap.Add(KinectStatus.Error, "Error");
this.statusMap.Add(KinectStatus.Initializing, "Initializing...");
this.statusMap.Add(KinectStatus.InsufficientBandwidth, "Insufficient Bandwidth");
this.statusMap.Add(KinectStatus.NotPowered, "Not Powered");
this.statusMap.Add(KinectStatus.NotReady, "Not Ready");
}
protected override void OnKinectSensorChanged(object sender, KinectSensorManagerEventArgs<KinectSensor> args)
{
if (null == args)
{
throw new ArgumentNullException("args");
}
if (null != args.OldValue)
{
args.OldValue.ColorFrameReady -= this.ColorImageReady;
if (!this.RetainImageOnSensorChange)
{
kinectColorImage.Source = null;
this.lastImageFormat = ColorImageFormat.Undefined;
}
}
if ((null != args.NewValue) && (KinectStatus.Connected == args.NewValue.Status))
{
ResetFrameRateCounters();
if (ColorImageFormat.RawYuvResolution640x480Fps15 == args.NewValue.ColorStream.Format)
{
throw new NotImplementedException("RawYuv conversion is not yet implemented.");
}
else
{
args.NewValue.ColorFrameReady += this.ColorImageReady;
}
}
}
private void KinectSensorOnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
using (var colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame())
{
if (colorImageFrame == null)
{
return;
}
// Make a copy of the color frame for displaying.
var haveNewFormat = this.currentColorImageFormat != colorImageFrame.Format;
if (haveNewFormat)
{
this.currentColorImageFormat = colorImageFrame.Format;
this.colorImageData = new byte[colorImageFrame.PixelDataLength];
this.colorImageWritableBitmap = new WriteableBitmap(
colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null);
ColorImage.Source = this.colorImageWritableBitmap;
}
colorImageFrame.CopyPixelDataTo(this.colorImageData);
this.colorImageWritableBitmap.WritePixels(
new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height),
this.colorImageData,
colorImageFrame.Width * Bgr32BytesPerPixel,
0);
/*
double length = Point.Subtract(faceTrackingViewer.ppLeft, faceTrackingViewer.ppRight).Length;
tB.Text = length.ToString();
if (length > 19)
{
elp.Fill = Brushes.Red;
}
else
{
elp.Fill = Brushes.Green;
}
*/
double mouthWidth = faceTrackingViewer.mouthWidth;
double noseWidth = faceTrackingViewer.noseWidth;
double threshold = noseWidth * modifyValue;
tBMouthDistance.Text = mouthWidth.ToString();
tbThreshold.Text = threshold.ToString();
if (mouthWidth > threshold)
{
elp.Fill = Brushes.Red;
}
else
{
elp.Fill = Brushes.Green;
}
}
}
/// <summary>
/// Map PointSkeleton3D to Point2D
/// </summary>
/// <param name="pointSkleton3D"></param>
/// <param name="colorImageFormat"></param>
/// <returns></returns>
public Point2D MapSkeletonPointToColorPoint(PointSkeleton3D pointSkleton3D, ColorImageFormat colorImageFormat)
{
SkeletonPoint point = new SkeletonPoint();
point.X = pointSkleton3D.X;
point.Y = pointSkleton3D.Y;
point.Z = pointSkleton3D.Z;
ColorImagePoint ImgPoint = mapper.MapSkeletonPointToColorPoint(point, colorImageFormat);
return new Point2D(ImgPoint.X, ImgPoint.Y);
}
/// <summary>
/// Map PointSkeleton3D List to Point2 List
/// </summary>
/// <param name="pointSkeleton3D"></param>
/// <param name="colorImageFormat"></param>
/// <returns></returns>
public List<Point2D> MapSkeletonPointsToColorPoints(List<PointSkeleton3D> pointSkeleton3D, ColorImageFormat colorImageFormat)
{
List<Point2D> ret = new List<Point2D>();
foreach (var element in pointSkeleton3D)
{
ret.Add(MapSkeletonPointToColorPoint(element, colorImageFormat));
}
return ret;
}
public Kinect(Game game, ColorImageFormat colorFormat, DepthImageFormat depthFormat)
: base(game)
{
this.colorImageFormat = colorFormat;
this.depthImageFormat = depthFormat;
KinectSensor.KinectSensors.StatusChanged += this.KinectSensors_StatusChanged;
this.DiscoverSensor();
}
public static int GetHeight(ColorImageFormat format)
{
switch (format)
{
case ColorImageFormat.RgbResolution1280x960Fps12:
return 960;
case ColorImageFormat.RawYuvResolution640x480Fps15:
case ColorImageFormat.RgbResolution640x480Fps30:
case ColorImageFormat.YuvResolution640x480Fps15:
return 480;
default:
return 480;
}
}
public MainWindow()
{
InitializeComponent();
faceTrackingViewerBinding = new Binding("Kinect") {Source = sensorChooser};
CurrentColorImageFormat = ColorImageFormat.Undefined;
Bgr32BytesPerPixel = (PixelFormats.Bgr32.BitsPerPixel + 7) / 8;
sensorChooser = new KinectSensorChooser();
sensorChooser.KinectChanged += SensorChooserOnKinectChanged;
sensorChooser.Start();
faceTrackingViewer.SetBinding(FaceTrackingViewer.KinectProperty, faceTrackingViewerBinding);
GetDirectory();
}
protected override void OnKinectChanged(KinectSensor oldKinectSensor, KinectSensor newKinectSensor)
{
if (oldKinectSensor != null) {
oldKinectSensor.ColorFrameReady -= ColorImageReady;
kinectColorImage.Source = null;
lastImageFormat = ColorImageFormat.Undefined;
}
if (newKinectSensor != null && newKinectSensor.Status == KinectStatus.Connected) {
ResetFrameRateCounters();
if (newKinectSensor.ColorStream.Format == ColorImageFormat.RawYuvResolution640x480Fps15) {
throw new NotImplementedException("RawYuv conversion is not yet implemented.");
}
else {
newKinectSensor.ColorFrameReady += ColorImageReady;
}
}
}
/// <summary>
/// Initializes a new instance of the KinectChooser class.
/// </summary>
/// <param name="game">The related game object.</param>
/// <param name="colorFormat">The desired color image format.</param>
/// <param name="depthFormat">The desired depth image format.</param>
public KinectChooser(Game game, ColorImageFormat colorFormat, DepthImageFormat depthFormat)
: base(game)
{
this.colorImageFormat = colorFormat;
this.depthImageFormat = depthFormat;
KinectSensor.KinectSensors.StatusChanged += this.KinectSensors_StatusChanged;
this.DiscoverSensor();
this.statusMap.Add(KinectStatus.Connected, string.Empty);
this.statusMap.Add(KinectStatus.DeviceNotGenuine, "Device Not Genuine");
this.statusMap.Add(KinectStatus.DeviceNotSupported, "Device Not Supported");
this.statusMap.Add(KinectStatus.Disconnected, "Required");
this.statusMap.Add(KinectStatus.Error, "Error");
this.statusMap.Add(KinectStatus.Initializing, "Initializing...");
this.statusMap.Add(KinectStatus.InsufficientBandwidth, "Insufficient Bandwidth");
this.statusMap.Add(KinectStatus.NotPowered, "Not Powered");
this.statusMap.Add(KinectStatus.NotReady, "Not Ready");
}
public KinectDevice()
{
//kinect設定
this.kinect = KinectSensor.GetDefault();
//設定とハンドラ
//colorImage
#region
this.colorImageFormat = ColorImageFormat.Bgra;
this.colorFrameDescription = this.kinect.ColorFrameSource.CreateFrameDescription(this.colorImageFormat);
this.colorFrameReader = this.kinect.ColorFrameSource.OpenReader();
this.colorFrameReader.FrameArrived += ColorFrame_Arrived;
this.colors = new byte[this.colorFrameDescription.Width
* this.colorFrameDescription.Height
* this.colorFrameDescription.BytesPerPixel];
#endregion
//骨格情報
#region
this.bodyFrameReader = this.kinect.BodyFrameSource.OpenReader();
this.bodyFrameReader.FrameArrived += BodyFrame_Arrived;
#endregion
//震度情報
#region
this.depthFrameReader = this.kinect.DepthFrameSource.OpenReader();
this.depthFrameReader.FrameArrived += DepthFrame_Arrived;
this.depthFrameDescription = this.kinect.DepthFrameSource.FrameDescription;
this.depthBuffer = new ushort[this.depthFrameDescription.LengthInPixels];
#endregion
//BodyIndex
#region
this.bodyIndexFrameDes = this.kinect.BodyIndexFrameSource.FrameDescription;
this.bodyIndexFrameReader = this.kinect.BodyIndexFrameSource.OpenReader();
this.bodyIndexFrameReader.FrameArrived += this.BodyIndexFrame_Arrived;
this.bodyIndexBuffer = new byte[this.bodyIndexFrameDes.Width *
this.bodyIndexFrameDes.Height * this.bodyIndexFrameDes.BytesPerPixel];
#endregion
//kinect開始
this.package = new ShadowPackage();
this.imageWidth = this.bodyIndexFrameDes.Width;
this.imageHeight = this.bodyIndexFrameDes.Height;
this.imageBytePerPixel = (int)this.bodyIndexFrameDes.BytesPerPixel;
this.kinectImage = new Mat(this.imageHeight, this.imageWidth, MatType.CV_8UC1);
this.kinect.Open();
}
private bool video_is_exist; //フレームを取得できたか
#endregion Fields
#region Constructors
public VideoFrameClass(KinectSensor sensor, ColorImageFormat imgFormat)
{
this.kinectSensor = sensor;
switch (imgFormat)
{
case ColorImageFormat.RgbResolution640x480Fps30:
this.FrameWidth = 640;
this.FrameHeight = 480;
break;
case ColorImageFormat.RgbResolution1280x960Fps12:
this.FrameWidth = 1280;
this.FrameHeight = 960;
break;
default:
throw new FormatException();
}
this.AllocateMemory();
}
/// <summary>
/// Get the depth image size from the color image format.
/// </summary>
/// <param name="imageFormat">The depth image format.</param>
/// <returns>The width and height of the color image format.</returns>
public static Size GetColorSize(ColorImageFormat imageFormat)
{
switch (imageFormat)
{
case ColorImageFormat.InfraredResolution640x480Fps30:
case ColorImageFormat.RawBayerResolution640x480Fps30:
case ColorImageFormat.RawYuvResolution640x480Fps15:
case ColorImageFormat.RgbResolution640x480Fps30:
case ColorImageFormat.YuvResolution640x480Fps15:
return new Size(640, 480);
case ColorImageFormat.RawBayerResolution1280x960Fps12:
case ColorImageFormat.RgbResolution1280x960Fps12:
return new Size(1280, 960);
case ColorImageFormat.Undefined:
return new Size(0, 0);
}
throw new ArgumentOutOfRangeException("imageFormat");
}
/// <summary>
/// Selects the color data from the color stream.
/// </summary>
/// <param name="source">The source observable.</param>
/// <returns>An observable sequence of colorData.</returns>
public static IObservable<byte[]> SelectColorData(this IObservable<ColorFrameArrivedEventArgs> source, byte[] frameData, ColorImageFormat colorImageFormat = ColorImageFormat.Bgra)
{
if (source == null) throw new ArgumentNullException("source");
return source.Select(_ =>
{
using (var frame = _.FrameReference.AcquireFrame())
{
if (frame == null) return frameData;
if (frame.RawColorImageFormat == ColorImageFormat.Bgra)
{
frame.CopyRawFrameDataToArray(frameData);
}
else
{
frame.CopyConvertedFrameDataToArray(frameData, colorImageFormat);
}
return frameData;
}
});
}
public ColorImageProcesser(MainWindow window, KinectSensor kinectDevice,
int depthDataLength, int colorDataLength,
int dWidht, int dHeight,
int cWidth, int cHeight,
DepthImageFormat dImageFormat, ColorImageFormat cImageFormat)
{
_windowUI = window;
_kinectDevice = kinectDevice;
depthPixelData = new short[depthDataLength];
colorPixelData = new byte[colorDataLength];
depthFrameWidth = dWidht;
depthFrameHeight = dHeight;
colorFrameWidth = cWidth;
colorFrameHeight = cHeight;
depthFrameStride = depthFrameWidth * BytesPerPixel;
colorFrameStride = colorFrameWidth * BytesPerPixel;
depthImageFormat = dImageFormat;
colorImageFormat = cImageFormat;
}
private KinectHelper(TransformSmoothParameters tsp, bool near, ColorImageFormat colorFormat, DepthImageFormat depthFormat)
{
_kinectSensor = KinectSensor.KinectSensors.FirstOrDefault(s => s.Status == KinectStatus.Connected);
if (_kinectSensor == null)
{
throw new Exception("No Kinect-Sensor found.");
}
if (near)
{
_kinectSensor.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated;
_kinectSensor.DepthStream.Range = DepthRange.Near;
_kinectSensor.SkeletonStream.EnableTrackingInNearRange = true;
}
_kinectSensor.SkeletonStream.Enable(tsp);
_kinectSensor.ColorStream.Enable(colorFormat);
_kinectSensor.DepthStream.Enable(depthFormat);
_kinectSensor.AllFramesReady += AllFramesReady;
_kinectSensor.Start();
DepthImageFormat = depthFormat;
ColorImageFormat = colorFormat;
}
/// <summary>
/// Initializes a new instance of the MainWindow class.
/// </summary>
public MainWindow()
{
// one sensor is currently supported
this.kinectSensor = KinectSensor.GetDefault();
// get the coordinate mapper
this.coordinateMapper = this.kinectSensor.CoordinateMapper;
// get the depth (display) extents
FrameDescription frameDescription = this.kinectSensor.DepthFrameSource.FrameDescription;
// get size of joint space
this.displayWidth = frameDescription.Width;
this.displayHeight = frameDescription.Height;
// open the reader for the body frames
//this.bodyFrameReader = this.kinectSensor.BodyFrameSource.OpenReader();
this.MSreader = this.kinectSensor.OpenMultiSourceFrameReader(FrameSourceTypes.Body | FrameSourceTypes.Color);
var fd = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Bgra);
uint frameSize = fd.BytesPerPixel * fd.LengthInPixels;
colorData = new byte[frameSize];
format = ColorImageFormat.Bgra;
imageSerial = 0;
// this.MSreader = this.kinectSensor.OpenMultiSourceFrameReader(FrameSourceTypes.Body | FrameSourceTypes.Color | FrameSourceTypes.Infrared);
// a bone defined as a line between two joints
// this.reader = this.kinectSensor.ColorFrameSource.OpenReader();
this.bones = new List <Tuple <JointType, JointType> >();
// Torso
this.bones.Add(new Tuple <JointType, JointType>(JointType.Head, JointType.Neck));
this.bones.Add(new Tuple <JointType, JointType>(JointType.Neck, JointType.SpineShoulder));
this.bones.Add(new Tuple <JointType, JointType>(JointType.SpineShoulder, JointType.SpineMid));
this.bones.Add(new Tuple <JointType, JointType>(JointType.SpineMid, JointType.SpineBase));
this.bones.Add(new Tuple <JointType, JointType>(JointType.SpineShoulder, JointType.ShoulderRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.SpineShoulder, JointType.ShoulderLeft));
this.bones.Add(new Tuple <JointType, JointType>(JointType.SpineBase, JointType.HipRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.SpineBase, JointType.HipLeft));
// Right Arm
this.bones.Add(new Tuple <JointType, JointType>(JointType.ShoulderRight, JointType.ElbowRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.ElbowRight, JointType.WristRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.WristRight, JointType.HandRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.HandRight, JointType.HandTipRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.WristRight, JointType.ThumbRight));
// Left Arm
this.bones.Add(new Tuple <JointType, JointType>(JointType.ShoulderLeft, JointType.ElbowLeft));
this.bones.Add(new Tuple <JointType, JointType>(JointType.ElbowLeft, JointType.WristLeft));
this.bones.Add(new Tuple <JointType, JointType>(JointType.WristLeft, JointType.HandLeft));
this.bones.Add(new Tuple <JointType, JointType>(JointType.HandLeft, JointType.HandTipLeft));
this.bones.Add(new Tuple <JointType, JointType>(JointType.WristLeft, JointType.ThumbLeft));
// Right Leg
this.bones.Add(new Tuple <JointType, JointType>(JointType.HipRight, JointType.KneeRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.KneeRight, JointType.AnkleRight));
this.bones.Add(new Tuple <JointType, JointType>(JointType.AnkleRight, JointType.FootRight));
// Left Leg
this.bones.Add(new Tuple <JointType, JointType>(JointType.HipLeft, JointType.KneeLeft));
this.bones.Add(new Tuple <JointType, JointType>(JointType.KneeLeft, JointType.AnkleLeft));
this.bones.Add(new Tuple <JointType, JointType>(JointType.AnkleLeft, JointType.FootLeft));
// populate body colors, one for each BodyIndex
this.bodyColors = new List <Pen>();
this.bodyColors.Add(new Pen(Brushes.Red, 6));
this.bodyColors.Add(new Pen(Brushes.Orange, 6));
this.bodyColors.Add(new Pen(Brushes.Green, 6));
this.bodyColors.Add(new Pen(Brushes.Blue, 6));
this.bodyColors.Add(new Pen(Brushes.Indigo, 6));
this.bodyColors.Add(new Pen(Brushes.Violet, 6));
// set IsAvailableChanged event notifier
this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;
// open the sensor
this.kinectSensor.Open();
// set the status text
this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
: Properties.Resources.NoSensorStatusText;
// Create the drawing group we'll use for drawing
this.drawingGroup = new DrawingGroup();
// Create an image source that we can use in our image control
this.imageSource = new DrawingImage(this.drawingGroup);
// use the window object as the view model in this simple example
this.DataContext = this;
//StartButton.Click += StartClicked;
// initialize the components (controls) of the window
this.InitializeComponent();
}
/// <summary>
/// Starts face tracking from Kinect input data. Track() detects a face
/// based on the passed parameters, then identifies characteristic
/// points and begins tracking. The first call to this API is more
/// expensive, but if the tracking succeeds then subsequent calls use
/// the tracking information generated from first call and is faster,
/// until a tracking failure happens.
/// </summary>
/// <param name="colorImageFormat">format of the colorImage array</param>
/// <param name="colorImage">Input color image frame retrieved from Kinect sensor</param>
/// <param name="depthImageFormat">format of the depthImage array</param>
/// <param name="depthImage">Input depth image frame retrieved from Kinect sensor</param>
/// <param name="skeletonOfInterest">Input skeleton to track. Head & shoulder joints in the skeleton are used to calculate the head vector</param>
/// <param name="regionOfInterest">Region of interest in the passed video frame where the face tracker should search for a face to initiate tracking.
/// Passing Rectangle.Empty (default) causes the entire frame to be searched.</param>
/// <returns>Returns computed face tracking results for this image frame</returns>
private FaceTrackFrame Track(
ColorImageFormat colorImageFormat,
byte[] colorImage,
DepthImageFormat depthImageFormat,
short[] depthImage,
Skeleton skeletonOfInterest,
Rect regionOfInterest)
{
this.totalTracks++;
this.trackStopwatch.Start();
if (this.operationMode != OperationMode.Kinect)
{
throw new InvalidOperationException(
"Cannot use Track with Kinect input types when face tracker is initialized for tracking videos/images");
}
if (colorImage == null)
{
throw new ArgumentNullException("colorImage");
}
if (depthImage == null)
{
throw new ArgumentNullException("depthImage");
}
if (colorImageFormat != this.initializationColorImageFormat)
{
throw new InvalidOperationException("Color image frame format different from initialization");
}
if (depthImageFormat != this.initializationDepthImageFormat)
{
throw new InvalidOperationException("Depth image frame format different from initialization");
}
if (colorImage.Length != this.videoCameraConfig.FrameBufferLength)
{
throw new ArgumentOutOfRangeException("colorImage", "Color image data size is needs to match initialization configuration.");
}
if (depthImage.Length != this.depthCameraConfig.FrameBufferLength)
{
throw new ArgumentOutOfRangeException("depthImage", "Depth image data size is needs to match initialization configuration.");
}
int hr;
HeadPoints headPointsObj = null;
Vector3DF[] headPoints = GetHeadPointsFromSkeleton(skeletonOfInterest);
if (headPoints != null && headPoints.Length == 2)
{
headPointsObj = new HeadPoints {
Points = headPoints
};
}
this.copyStopwatch.Start();
this.colorFaceTrackingImage.CopyFrom(colorImage);
this.depthFaceTrackingImage.CopyFrom(depthImage);
this.copyStopwatch.Stop();
var sensorData = new SensorData(this.colorFaceTrackingImage, this.depthFaceTrackingImage, DefaultZoomFactor, Point.Empty);
FaceTrackingSensorData faceTrackSensorData = sensorData.FaceTrackingSensorData;
this.startOrContinueTrackingStopwatch.Start();
if (this.trackSucceeded)
{
hr = this.faceTrackerInteropPtr.ContinueTracking(ref faceTrackSensorData, headPointsObj, this.frame.ResultPtr);
}
else
{
hr = this.faceTrackerInteropPtr.StartTracking(
ref faceTrackSensorData, ref regionOfInterest, headPointsObj, this.frame.ResultPtr);
}
this.startOrContinueTrackingStopwatch.Stop();
this.trackSucceeded = hr == (int)ErrorCode.Success && this.frame.Status == ErrorCode.Success;
this.trackStopwatch.Stop();
if (this.trackSucceeded)
{
++this.totalSuccessTracks;
this.totalSuccessTrackMs += this.trackStopwatch.ElapsedMilliseconds - this.lastSuccessTrackElapsedMs;
this.lastSuccessTrackElapsedMs = this.trackStopwatch.ElapsedMilliseconds;
}
return(this.frame);
}
private static extern IntPtr Windows_Kinect_ColorFrameSource_CreateFrameDescription(IntPtr pNative, ColorImageFormat format);
/// <summary>
/// Map PointSkeleton3D List to Point2 List
/// </summary>
/// <param name="pointSkeleton3D"></param>
/// <param name="colorImageFormat"></param>
/// <returns></returns>
public List <Point2D> MapSkeletonPointsToColorPoints(List <PointSkeleton3D> pointSkeleton3D, ColorImageFormat colorImageFormat)
{
List <Point2D> ret = new List <Point2D>();
foreach (var element in pointSkeleton3D)
{
ret.Add(MapSkeletonPointToColorPoint(element, colorImageFormat));
}
return(ret);
}
void nui_AllFramesReady(object sender, AllFramesReadyEventArgs e)
{
ColorImageFrame cf = e.OpenColorImageFrame();
DepthImageFrame df = e.OpenDepthImageFrame();
SkeletonFrame sf = e.OpenSkeletonFrame();
// Skeleton[] skeletonData = new Skeleton[sf.SkeletonArrayLength];
if (cf == null || df == null || sf == null)
{
return;
}
byte[] ImageBits = new byte[cf.PixelDataLength];
cf.CopyPixelDataTo(ImageBits);
BitmapSource src = null;
src = BitmapSource.Create(cf.Width, cf.Height,
96, 96, PixelFormats.Bgr32, null,
ImageBits,
cf.Width * cf.BytesPerPixel);
image2.Source = src;
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != df.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = df.Format;
}
if (this.colorImageFormat != cf.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = cf.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[df.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[cf.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != sf.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[sf.SkeletonArrayLength];
}
cf.CopyPixelDataTo(this.colorImage);
df.CopyPixelDataTo(this.depthImage);
sf.CopySkeletonDataTo(this.skeletonData);
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(nui2, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = sf.FrameNumber;
}
}
}
using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame())
{
if (depthImageFrame != null)
{
foreach (Skeleton sd in skeletonData)
{
if (sd.TrackingState == SkeletonTrackingState.Tracked)
{
Joint joint = sd.Joints[JointType.Head];
DepthImagePoint depthPoint;
// CoordinateMapper coordinateMapper = new CoordinateMapper(nui);
// depthPoint = coordinateMapper.MapSkeletonPointToDepthPoint(joint.Position, DepthImageFormat.Resolution320x240Fps30);
depthPoint = depthImageFrame.MapFromSkeletonPoint(joint.Position);
System.Windows.Point point = new System.Windows.Point((int)(image2.ActualWidth * depthPoint.X
/ depthImageFrame.Width),
(int)(image2.ActualHeight * depthPoint.Y
/ depthImageFrame.Height));
Canvas.SetLeft(ellipse1, (point.X) - ellipse1.Width);
Canvas.SetTop(ellipse1, (point.Y) - ellipse1.Height);
App thisApp = App.Current as App;
Canvas.SetLeft(rect2, thisApp.m_dbX - rect2.Width);
Canvas.SetTop(rect2, thisApp.m_dbY - rect2.Height);
double GapX, GapY;
GapX = point.X - (thisApp.m_dbX - 2);
GapY = point.Y - (thisApp.m_dbY - 2);
int siteX = 999, siteY = 999;
if (GapX < 30 && GapX > -30)
{
siteX = 1;
}
else if (GapX >= 30)
{
siteX = 0;
}
else if (GapY <= -30)
{
siteX = 2;
}
if (GapY >= -40)
{
siteY = 0;
}
else if (GapY < -40 && GapY > -60)
{
siteY = 1;
}
else if (GapY <= -60)
{
siteY = 2;
}
int site;
site = siteX + (siteY * 3);
if (site == 0)
{
text2.Text = "좌상";
}
else if (site == 1)
{
text2.Text = "상";
}
else if (site == 2)
{
text2.Text = "우상";
}
else if (site == 3)
{
text2.Text = "좌";
}
else if (site == 4)
{
text2.Text = "정";
}
else if (site == 5)
{
text2.Text = "우";
}
else if (site == 6)
{
text2.Text = "좌하";
}
else if (site == 7)
{
text2.Text = "하";
}
else if (site == 8)
{
text2.Text = "우하";
}
thisApp.nowsite = site;
/*
*
* rect4.X = facePoints[i].X - 2;
* rect4.Y = facePoints[i].Y - 2;
* rect4.Width = 4;
* rect4.Height = 4;
*/
}
}
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
///
public void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
//No Touchy
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
return;
}
if (faceTracker == null)
{
faceTracker = new FaceTracker(kinectSensor);
}
frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
faceTriangles = frame.GetTriangles();
}
this.facePoints = frame.GetProjected3DShape();
//Touchy
//Assign Reference points
this.absfacePoints = frame.Get3DShape();
leftForehead = this.absfacePoints[FeaturePoint.TopLeftForehead];
rightForehead = this.absfacePoints[FeaturePoint.TopRightForehead];
jaw = this.absfacePoints[FeaturePoint.BottomOfChin];
faceRotationX = frame.Rotation.X;
faceRotationY = frame.Rotation.Y;
faceRotationZ = frame.Rotation.Z;
//Calculate Reference Points
foreheadReferencePointX = ((rightForehead.X - leftForehead.X) / 2);
foreheadReferencePointY = ((rightForehead.Y - leftForehead.Y) / 2);
foreheadReferencePointZ = ((rightForehead.Z - leftForehead.Z) / 2);
//Set Animation Units
AUCoeff = frame.GetAnimationUnitCoefficients();
jawLowererAU = AUCoeff[AnimationUnit.JawLower];
lipStretcherAU = AUCoeff[AnimationUnit.LipStretcher];
browRaiserAU = AUCoeff[AnimationUnit.BrowRaiser];
setJawData(jaw.Y, leftForehead.Y, rightForehead.Y, jawLowererAU, lipStretcherAU);
rotations = new float[5];
//set up matlab
matlab = new MLApp.MLApp();
matlab.Execute(@"cd C:\Users\Bala\Documents\MATLAB");
result = null;
//get rotation values
rotations[0] = faceRotationX;
rotations[1] = faceRotationY;
rotations[2] = faceRotationZ;
rotations[3] = jawLowererAU;
rotations[4] = lipStretcherAU;
//Set up GlovePie
OscPacket.LittleEndianByteOrder = false;
IPEndPoint myapp = new IPEndPoint(IPAddress.Loopback, 1944);
IPEndPoint glovepie = new IPEndPoint(IPAddress.Loopback, 1945);
Console.WriteLine(browRaiserAU);
matlab.Feval("nnW", 1, out result, rotations[0]);
object[] resW = result as object[];
nnoutput = (int)((float)resW[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 1;
}
else
{
result = null;
matlab.Feval("nnA", 1, out result, rotations[1]);
object[] resA = result as object[];
nnoutput = (int)((float)resA[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 2;
}
else
{
result = null;
matlab.Feval("nnS", 1, out result, rotations[0]);
object[] resS = result as object[];
nnoutput = (int)((float)resS[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 3;
}
else
{
result = null;
matlab.Feval("nnd", 1, out result, rotations[1]);
object[] resD = result as object[];
nnoutput = (int)((float)resD[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 4;
}
else
{
result = null;
matlab.Feval("nnLC", 1, out result, rotations[2]);
object[] resLC = result as object[];
nnoutput = (int)((float)resLC[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 5;
}
else
{
result = null;
matlab.Feval("nnRC", 1, out result, rotations[2]);
object[] resRC = result as object[];
nnoutput = (int)((float)resRC[0] + 0.5f);
if (nnoutput == 1)
{
commandtoSend = 6;
}
else
{
result = null;
if (jawLowererAU > 0.7)
{
commandtoSend = 7;
}
/*
* matlab.Feval("nnSpace", 1, out result, rotations[3]);
* object[] resSpace = result as object[];
* nnoutput = (int)((float)resSpace[0] + 0.5f);
* if (nnoutput == 1)
* {
* commandtoSend = 7;
* }*/
else
{
result = null;
if (browRaiserAU > 0.4)
{
commandtoSend = 8;
}
else
{
result = null;
commandtoSend = 0;
}
/*result = null;
* matlab.Feval("nnMiddle", 1, out result, lipStretcherAU);
* object[] resMiddle = result as object[];
* nnoutput = (int)((float)resMiddle[0] + 0.5f);
* if (nnoutput == 1)
* {
* commandtoSend = 8;
* }
* else
* {
* result = null;
* commandtoSend = 0;
* }*/
}
}
}
}
}
}
}
//Console.WriteLine("Iteration Complete");
switch (commandtoSend)
{
case 0:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 1:
msg = new OscMessage(myapp, "/move/w", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 2:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 3:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 4:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 5:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 6:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 7:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 10.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 0.0f);
msg.Send(glovepie);
break;
case 8:
msg = new OscMessage(myapp, "/move/w", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/a", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/s", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/d", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/lc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/rc", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/space", 0.0f);
msg.Send(glovepie);
msg = new OscMessage(myapp, "/move/middle", 10.0f);
msg.Send(glovepie);
break;
}
}
}
/// <summary>
/// Selects the color data from the color stream.
/// </summary>
/// <param name="source">The source observable.</param>
/// <returns>An observable sequence of colorData.</returns>
public static IObservable <byte[]> SelectColorData(this IObservable <ColorFrameArrivedEventArgs> source, byte[] frameData, ColorImageFormat colorImageFormat = ColorImageFormat.Bgra)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
return(source.Select(_ =>
{
using (var frame = _.FrameReference.AcquireFrame())
{
if (frame == null)
{
return frameData;
}
if (frame.RawColorImageFormat == ColorImageFormat.Bgra)
{
frame.CopyRawFrameDataToArray(frameData);
}
else
{
frame.CopyConvertedFrameDataToArray(frameData, colorImageFormat);
}
return frameData;
}
}));
}
private void ColorImageReady(object sender, ColorImageFrameReadyEventArgs e)
{
using (ColorImageFrame imageFrame = e.OpenColorImageFrame())
{
if (imageFrame != null)
{
// We need to detect if the format has changed.
bool haveNewFormat = this.lastImageFormat != imageFrame.Format;
bool convertToRgb = false;
int bytesPerPixel = imageFrame.BytesPerPixel;
if (imageFrame.Format == ColorImageFormat.RawBayerResolution640x480Fps30 ||
imageFrame.Format == ColorImageFormat.RawBayerResolution1280x960Fps12)
{
convertToRgb = true;
bytesPerPixel = 4;
}
if (haveNewFormat)
{
if (convertToRgb)
{
this.rawPixelData = new byte[imageFrame.PixelDataLength];
this.pixelData = new byte[bytesPerPixel * imageFrame.Width * imageFrame.Height];
}
else
{
this.pixelData = new byte[imageFrame.PixelDataLength];
}
}
if (convertToRgb)
{
imageFrame.CopyPixelDataTo(this.rawPixelData);
ConvertBayerToRgb32(imageFrame.Width, imageFrame.Height);
}
else
{
imageFrame.CopyPixelDataTo(this.pixelData);
}
// A WriteableBitmap is a WPF construct that enables resetting the Bits of the image.
// This is more efficient than creating a new Bitmap every frame.
if (haveNewFormat)
{
PixelFormat format = PixelFormats.Bgr32;
if (imageFrame.Format == ColorImageFormat.InfraredResolution640x480Fps30)
{
format = PixelFormats.Gray16;
}
kinectColorImage.Visibility = Visibility.Visible;
this.outputImage = new WriteableBitmap(
imageFrame.Width,
imageFrame.Height,
96, // DpiX
96, // DpiY
format,
null);
this.kinectColorImage.Source = this.outputImage;
}
this.outputImage.WritePixels(
new Int32Rect(0, 0, imageFrame.Width, imageFrame.Height),
this.pixelData,
imageFrame.Width * bytesPerPixel,
0);
this.lastImageFormat = imageFrame.Format;
UpdateFrameRate();
}
}
}
private void DecodeSkeletonData(AllFramesReadyEventArgs e, KinectSensor sensor)
{
#region GetImageFormat
ColorImageFormat colorFormat = ColorImageFormat.Undefined;
int colorWidth = 0;
int colorHeight = 0;
DepthImageFormat depthFormat = DepthImageFormat.Undefined;
int depthWidth = 0;
int depthHeight = 0;
switch (this._imageType)
{
case ImageType.Color:
// Retrieve the current color format, from the frame if present, and from the sensor if not.
using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame())
{
if (null != colorImageFrame)
{
colorFormat = colorImageFrame.Format;
colorWidth = colorImageFrame.Width;
colorHeight = colorImageFrame.Height;
}
else if (null != sensor.ColorStream)
{
colorFormat = sensor.ColorStream.Format;
colorWidth = sensor.ColorStream.FrameWidth;
colorHeight = sensor.ColorStream.FrameHeight;
}
}
break;
case ImageType.Depth:
// Retrieve the current depth format, from the frame if present, and from the sensor if not.
using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame())
{
if (null != depthImageFrame)
{
depthFormat = depthImageFrame.Format;
depthWidth = depthImageFrame.Width;
depthHeight = depthImageFrame.Height;
}
else if (null != sensor.DepthStream)
{
depthFormat = sensor.DepthStream.Format;
depthWidth = sensor.DepthStream.FrameWidth;
depthHeight = sensor.DepthStream.FrameHeight;
}
}
break;
}
#endregion
// Clear the play canvas
this.playField.Children.Clear();
// Check every skeleton
for (int skeletonSlot = 0; skeletonSlot < this._skeletonBuffer.Length; skeletonSlot++)
{
var skeleton = this._skeletonBuffer[skeletonSlot];
#region Skeleton Position
// Map points between skeleton and color/depth
var jointMapping = this._jointMappings[skeletonSlot];
jointMapping.Clear();
try
{
// Transform the data into the correct space
// For each joint, we determine the exact X/Y coordinates for the target view
foreach (Joint joint in skeleton.Joints)
{
ColorImagePoint colorPoint = sensor.CoordinateMapper.MapSkeletonPointToColorPoint(joint.Position, colorFormat);
Point mappedPoint = new Point(
(int)(this._renderSize.Width * colorPoint.X / colorWidth),
(int)(this._renderSize.Height * colorPoint.Y / colorHeight));
jointMapping[joint.JointType] = new JointMapping
{
Joint = joint,
MappedPoint = mappedPoint,
OriginPoint = colorPoint,
};
}
}
catch (UnauthorizedAccessException)
{
// Kinect is no longer available.
return;
}
// Look up the center point
Point centerPoint = PositionCalculator.Get2DPosition(
sensor,
this._imageType,
this._renderSize,
skeleton.Position,
colorFormat,
colorWidth,
colorHeight,
depthFormat,
depthWidth,
depthHeight);
#endregion
// Scale the skeleton thickness
// 1.0 is the desired size at 640 width
this._scaleFactor = this._renderSize.Width / colorWidth;
// Displays a gradient near the edge of the display
// where the skeleton is leaving the screen
this.DrawClippedEdges(skeleton);
switch (skeleton.TrackingState)
{
case SkeletonTrackingState.PositionOnly:
{
// The skeleton is being tracked, but we only know the general position, and
// we do not know the specific joint locations.
this.DrawBodyCenter(centerPoint);
}
break;
case SkeletonTrackingState.Tracked:
{
// The skeleton is being tracked and the joint data is available for consumption.
this.DrawBody(skeleton, jointMapping);
// Track player
this.TrackPlayer(skeleton, jointMapping);
}
break;
}
}
}
private static extern RootSystem.IntPtr Windows_Kinect_ColorFrame_CreateFrameDescription(
RootSystem.IntPtr pNative, ColorImageFormat format);
private static extern void Windows_Kinect_ColorFrame_CopyConvertedFrameDataToArray(RootSystem.IntPtr pNative,
RootSystem.IntPtr frameData, int frameDataSize, ColorImageFormat colorFormat);
private void KinectAllFramesReady(object sender, AllFramesReadyEventArgs e)
{
KinectSensor sensor = sender as KinectSensor;
foreach (var skeletonCanvas in this.skeletonCanvases)
{
skeletonCanvas.Skeleton = null;
}
// Have we already been "shut down" by the user of this viewer,
// or has the SkeletonStream been disabled since this event was posted?
if ((null == this.KinectSensorManager) ||
(null == sensor) ||
(null == sensor.SkeletonStream) ||
!sensor.SkeletonStream.IsEnabled)
{
return;
}
bool haveSkeletonData = false;
long frameTimeStamp = -1;
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
{
if (skeletonFrame != null)
{
if ((this.skeletonData == null) || (this.skeletonData.Length != skeletonFrame.SkeletonArrayLength))
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
frameTimeStamp = skeletonFrame.Timestamp;
haveSkeletonData = true;
}
}
int trackedIndex = -1;
// find the first tracked skeleton and set var trackedSkeleton accordingly
for (int i = 0; i < skeletonData.Length; i++)
{
if (skeletonData[i].TrackingState.Equals(SkeletonTrackingState.Tracked))
{
trackedIndex = i;
break;
}
}
bool isFullyTracked = false;
if (isMeasuring && trackedIndex > -1)
{
// check to see if the skeleton @ trackedIndex is fully tracked
if (fullyTrackedMapping == null && IsFullyTracked(skeletonData[trackedIndex]))
{
isFullyTracked = true;
}
SkeletonMeasurer measurer = new SkeletonMeasurer(skeletonData[trackedIndex]);
measurer.determineMeasurements();
AddMeasurementsToBuffer(measurer.TestMeasurements);
skeletonBuffer.Add(ObjectCopier.Clone <Skeleton>(skeletonData[trackedIndex]));
frameTimeStampBuffer.Add(frameTimeStamp);
}
if (haveSkeletonData)
{
ColorImageFormat colorFormat = ColorImageFormat.Undefined;
int colorWidth = 0;
int colorHeight = 0;
DepthImageFormat depthFormat = DepthImageFormat.Undefined;
int depthWidth = 0;
int depthHeight = 0;
switch (this.ImageType)
{
case ImageType.Color:
// Retrieve the current color format, from the frame if present, and from the sensor if not.
using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame())
{
if (null != colorImageFrame)
{
colorFormat = colorImageFrame.Format;
colorWidth = colorImageFrame.Width;
colorHeight = colorImageFrame.Height;
}
else if (null != sensor.ColorStream)
{
colorFormat = sensor.ColorStream.Format;
colorWidth = sensor.ColorStream.FrameWidth;
colorHeight = sensor.ColorStream.FrameHeight;
}
}
break;
case ImageType.Depth:
// Retrieve the current depth format, from the frame if present, and from the sensor if not.
using (DepthImageFrame depthImageFrame = e.OpenDepthImageFrame())
{
if (null != depthImageFrame)
{
depthFormat = depthImageFrame.Format;
depthWidth = depthImageFrame.Width;
depthHeight = depthImageFrame.Height;
}
else if (null != sensor.DepthStream)
{
depthFormat = sensor.DepthStream.Format;
depthWidth = sensor.DepthStream.FrameWidth;
depthHeight = sensor.DepthStream.FrameHeight;
}
}
break;
}
for (int i = 0; i < this.skeletonData.Length && i < this.skeletonCanvases.Count; i++)
{
var skeleton = this.skeletonData[i];
var skeletonCanvas = this.skeletonCanvases[i];
var jointMapping = this.jointMappings[i];
jointMapping.Clear();
try
{
// Transform the data into the correct space
// For each joint, we determine the exact X/Y coordinates for the target view
foreach (Joint joint in skeleton.Joints)
{
Point mappedPoint = Get2DPosition(
sensor,
this.ImageType,
this.RenderSize,
joint.Position,
colorFormat,
colorWidth,
colorHeight,
depthFormat,
depthWidth,
depthHeight);
jointMapping[joint.JointType] = new JointMapping
{
Joint = joint,
MappedPoint = mappedPoint
};
}
}
catch (UnauthorizedAccessException)
{
// Kinect is no longer available.
return;
}
// Look up the center point
Point centerPoint = Get2DPosition(
sensor,
this.ImageType,
this.RenderSize,
skeleton.Position,
colorFormat,
colorWidth,
colorHeight,
depthFormat,
depthWidth,
depthHeight);
// Scale the skeleton thickness
// 1.0 is the desired size at 640 width
double scale = this.RenderSize.Width / 640;
skeletonCanvas.Skeleton = skeleton;
skeletonCanvas.JointMappings = jointMapping;
skeletonCanvas.Center = centerPoint;
skeletonCanvas.ScaleFactor = scale;
}
if (isFullyTracked)
{
fullyTrackedMapping = new Dictionary <JointType, JointMapping>();
foreach (JointType type in jointMappings[trackedIndex].Keys)
{
fullyTrackedMapping[type] = new JointMapping();
fullyTrackedMapping[type].Joint = jointMappings[trackedIndex][type].Joint;
fullyTrackedMapping[type].MappedPoint = jointMappings[trackedIndex][type].MappedPoint;
}
}
}
}
private void Kinect_AllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
if (skeletonFaceTracker.lastFaceTrackSucceeded)
{
if (skeletonFaceTracker.LastTrackedFrame % 10 == 0)
{
Out.Post(true, pipeline.GetCurrentTime());
}
}
else
{
if (skeletonFaceTracker.LastTrackedFrame % 10 == 0)
{
Out.Post(false, pipeline.GetCurrentTime());
}
}
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
//this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest, FaceRecognitionActivityWindow win)
{
if (CheckFace(kinectSensor, colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest))
{
count++;
}
else
{
count = 0;
}
if (count == 1)
{
count = 0;
currentState = (currentState + 1) % 3;
// highlight the next exercise
if (currentState == 0)
{
Color tileFill = Color.FromRgb(76, 76, 76);
SolidColorBrush brush1 = new SolidColorBrush(tileFill);
win.SadTile.Fill = brush1;
Color focusTileFill = Color.FromRgb(96, 96, 96);
SolidColorBrush brush2 = new SolidColorBrush(focusTileFill);
win.HappyTile.Fill = brush2;
Color secTileFill = Color.FromRgb(76, 76, 76);
SolidColorBrush brush3 = new SolidColorBrush(tileFill);
win.AngryTile.Fill = brush3;
// display the large icon
win.ActivityImage.Source = new BitmapImage(new Uri(@"happy_big.png"));
win.ActivityLabel.Content = "Happy";
}
else if (currentState == 1)
{
Color tileFill = Color.FromRgb(96, 96, 96);
SolidColorBrush brush1 = new SolidColorBrush(tileFill);
win.SadTile.Fill = brush1;
Color focusTileFill = Color.FromRgb(76, 76, 76);
SolidColorBrush brush2 = new SolidColorBrush(focusTileFill);
win.HappyTile.Fill = brush2;
Color secTileFill = Color.FromRgb(76, 76, 76);
SolidColorBrush brush3 = new SolidColorBrush(tileFill);
win.AngryTile.Fill = brush3;
// display the large icon
win.ActivityImage.Source = new BitmapImage(new Uri(@"sad_big.png"));
win.ActivityLabel.Content = "Sad";
}
else if (currentState == 2)
{
Color tileFill = Color.FromRgb(76, 76, 76);
SolidColorBrush brush1 = new SolidColorBrush(tileFill);
win.SadTile.Fill = brush1;
Color focusTileFill = Color.FromRgb(76, 76, 76);
SolidColorBrush brush2 = new SolidColorBrush(focusTileFill);
win.HappyTile.Fill = brush2;
Color secTileFill = Color.FromRgb(96, 96, 96);
SolidColorBrush brush3 = new SolidColorBrush(tileFill);
win.AngryTile.Fill = brush3;
// display the large icon
win.ActivityImage.Source = new BitmapImage(new Uri(@"angry_big.png"));
win.ActivityLabel.Content = "Angry";
}
this.speech.SpeakAsync("Moving to next level");
// Notify to change face
Trace.WriteLine("Change state to: " + states[currentState]);
}
}
private static extern void Windows_Kinect_ColorFrame_CopyConvertedFrameDataToIntPtr(IntPtr pNative, IntPtr frameData, uint frameDataSize, ColorImageFormat colorFormat);
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// if the current skeleton is not tracked, track it now
//kinectSensor.SkeletonStream.ChooseSkeletons(skeletonOfInterest.TrackingId);
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
// hack to make this face tracking detect the face even when it is not actually tracked
// <!>need to confirm if it works
//skeletonOfInterest.TrackingState = SkeletonTrackingState.Tracked;
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
//new Microsoft.Kinect.Toolkit.FaceTracking.Rect(skeletonOfInterest.Position.));
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
if (faceTag == null)
{
// here call the face detection
faceTag = new FaceRecognizer().getFaceTag(this.colorImageBmp);
if (faceTag != null)
{
Global.StatusBarText.Text = "Found " + faceTag + "!";
if (Global.trackedPeople.ContainsKey(skeletonOfInterest))
{
Global.trackedPeople[skeletonOfInterest] = faceTag;
}
else
{
Global.trackedPeople.Add(skeletonOfInterest, faceTag);
}
}
}
this.facePoints = frame.GetProjected3DShape();
this.faceRect = frame.FaceRect;
}
}
}
public ConvertedColorImage(ColorFrame frame, ColorImageFormat colorImageFormat)
{
this.frame = frame;
this.colorImageFormat = colorImageFormat;
Info = frame.FrameDescription.ToImageInfo(colorImageFormat);
}
/// <summary>
/// Execute startup tasks
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void WindowLoaded(object sender, RoutedEventArgs e)
{
// Look through all sensors and start the first connected one.
// This requires that a Kinect is connected at the time of app startup.
// To make your app robust against plug/unplug,
// it is recommended to use KinectSensorChooser provided in Microsoft.Kinect.Toolkit (See components in Toolkit Browser).
foreach (var potentialSensor in KinectSensor.KinectSensors)
{
if (potentialSensor.Status == KinectStatus.Connected)
{
this.sensor = potentialSensor;
break;
}
}
if (this.sensor != null)
{
// Turn on the skeleton, color, depth stream to receive skeleton frames
TransformSmoothParameters smoothingParam = new TransformSmoothParameters();
{
smoothingParam.Smoothing = 0.5f;
smoothingParam.Correction = 0.1f;
smoothingParam.Prediction = 0.5f;
smoothingParam.JitterRadius = 0.1f;
smoothingParam.MaxDeviationRadius = 0.1f;
};
this.sensor.SkeletonStream.Enable(smoothingParam);
//this.sensor.SkeletonStream.Enable();
this.sensor.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated;
this.checkBoxSeatedMode.SetCurrentValue(CheckBox.IsCheckedProperty, true);
this.sensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
this.sensor.DepthStream.Enable(DepthImageFormat.Resolution640x480Fps30);
this.RenderHeight = 480;
this.RenderWidth = 640;
this.depthImageFormat = this.sensor.DepthStream.Format;
this.colorImageFormat = this.sensor.ColorStream.Format;
// Create the drawing group we'll use for drawing
this.drawingGroup = new DrawingGroup();
this.drawingGroup.ClipGeometry = new RectangleGeometry(new Rect(0.0, 0.0, RenderWidth, RenderHeight));
this.outputDrawingGroup = new DrawingGroup();
this.outputDrawingGroup.ClipGeometry = new RectangleGeometry(new Rect(0.0, 0.0, RenderWidth, RenderHeight));
// Display the drawing using our image control
Image.Source = new DrawingImage(this.drawingGroup);
// Allocate space to put the pixels we'll receive
this.colorImage = new byte[this.sensor.ColorStream.FramePixelDataLength];
// This is the bitmap we'll display on-screen
this.colorBitmap = new WriteableBitmap(this.sensor.ColorStream.FrameWidth, this.sensor.ColorStream.FrameHeight, 96.0, 96.0, PixelFormats.Bgr32, null);
this.blankColorBitmap = new WriteableBitmap(this.sensor.ColorStream.FrameWidth, this.sensor.ColorStream.FrameHeight, 96.0, 96.0, PixelFormats.Bgr32, null);
OutputImage.Source = new DrawingImage(this.outputDrawingGroup);
RoomSetting.SetCameraMatrix();
RoomSetting.SetPlates();
// Add an event handler to be called whenever there is new all frame data
this.sensor.AllFramesReady += this.OnAllFramesReady;
// Start the sensor!
try
{
this.sensor.Start();
}
catch (IOException)
{
this.sensor = null;
}
}
if (null == this.sensor)
{
this.statusBarText.Text = Properties.Resources.NoKinectReady;
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
this.facePoints3DRaw = frame.Get3DShape();
this.facePoints = frame.GetProjected3DShape();
animationUnitsRaw = frame.GetAnimationUnitCoefficients();
}
x = frame.Rotation.X;
y = frame.Rotation.Y;
z = frame.Rotation.Z;
facePointS3D = this.facePoints3DRaw;
animationUnits = animationUnitsRaw;
//Debug.WriteLine(animationUnits[AnimationUnit.JawLower]);
//Debug.WriteLine(animationUnits[AnimationUnit.BrowLower]);
//Debug.WriteLine(animationUnits[AnimationUnit.BrowRaiser]);
//Debug.WriteLine(animationUnits[AnimationUnit.JawLower]);
//Debug.WriteLine(animationUnits[AnimationUnit.LipCornerDepressor]);
//Debug.WriteLine(animationUnits[AnimationUnit.LipRaiser]);
//Debug.WriteLine(animationUnits[AnimationUnit.LipStretcher]);
//Debug.WriteLine(frame.Translation.ToString());
//Debug.WriteLine(frame.Rotation.ToString());
//this.facePoints[FeaturePoint.AboveChin].X+2;
//Debug.WriteLine(frame.Translation.X.ToString());
//Debug.WriteLine(frame.Translation.Y.ToString());
//Debug.WriteLine(frame.Translation.Z.ToString());
}
}
/// <summary>
/// Maps every point in a depth frame to the corresponding location in a ColorImageFormat coordinate space.
///
/// </summary>
/// <param name="depthImageFormat">The depth format of the source.</param><param name="depthPixelData">The depth frame pixel data, as retrieved from DepthImageFrame.CopyPixelDataTo.
/// Must be equal in length to Width*Height of the depth format specified by depthImageFormat.
/// </param><param name="colorImageFormat">The desired target image format.</param><param name="colorCoordinates">The ColorImagePoint array to receive the data. Each element will be be the result of mapping the
/// corresponding depthPixelDatum to the specified ColorImageFormat coordinate space.
/// Must be equal in length to depthPixelData.
/// </param>
public void MapDepthFrameToColorFrame(DepthImageFormat depthImageFormat, short[] depthPixelData,
ColorImageFormat colorImageFormat, ColorImagePoint[] colorCoordinates)
{
_kinectSensor.MapDepthFrameToColorFrame(depthImageFormat, depthPixelData, colorImageFormat, colorCoordinates);
}
private void OnFrameReady(object sender, AllFramesReadyEventArgs e)
{
var imageWith = 0;
var imageHeight = 0;
var bytesPerPixel = 0;
var convertToRgb = false;
var haveNewFormat = false;
using (var frame = e.OpenColorImageFrame())
{
if (frame == null)
{
return;
}
imageWith = frame.Width;
imageHeight = frame.Height;
bytesPerPixel = frame.BytesPerPixel;
haveNewFormat = _lastImageFormat != frame.Format;
if (frame.Format == ColorImageFormat.RawBayerResolution640x480Fps30 ||
frame.Format == ColorImageFormat.RawBayerResolution1280x960Fps12)
{
convertToRgb = true;
bytesPerPixel = 4;
}
if (haveNewFormat)
{
_lastImageFormat = frame.Format;
_rawPixelData = new byte[frame.PixelDataLength];
_pixelData = new byte[frame.Width * frame.Height * bytesPerPixel];
}
if (convertToRgb)
{
frame.CopyPixelDataTo(_rawPixelData);
ConvertBayerToRgb32(imageWith, imageHeight, _rawPixelData, _pixelData);
}
else
{
frame.CopyPixelDataTo(_pixelData);
}
View.Dispatcher.Invoke(() =>
{
if (haveNewFormat)
{
_bitmap = new WriteableBitmap(imageWith, imageHeight, 96, 96, PixelFormats.Bgr32,
null);
//View.ColorImage.Source = _bitmap;
}
_bitmap.WritePixels(new Int32Rect(0, 0, imageWith, imageHeight), _pixelData, imageWith * bytesPerPixel, 0);
var bitmapSource = BitmapSource.Create(imageWith, imageHeight, 96, 96, PixelFormats.Bgr32, null, _pixelData, imageWith * 4);
var b = bitmapSource.ToBitmap();
var image = new Image <Bgr, byte>(b);
var grayImage = image.Convert <Gray, byte>();
using (var storage = new MemStorage())
{
var contours = grayImage.FindContours(
Emgu.CV.CvEnum.CHAIN_APPROX_METHOD.CV_CHAIN_APPROX_SIMPLE,
Emgu.CV.CvEnum.RETR_TYPE.CV_RETR_EXTERNAL,
storage);
for (var i = 0; contours != null; contours = contours.HNext)
{
i++;
if ((contours.Area > Math.Pow(10, 2)) && (contours.Area < Math.Pow(1000, 2)))
{
var box = contours.GetMinAreaRect();
image.Draw(box, new Bgr(System.Drawing.Color.Red), 2);
}
}
}
View.ColorImage.Source = image.ToBitmapSource();
});
/*for (var y = 0; y < frame.Height; y++)
* {
* for (var x = 0; x < frame.Width; x++)
* {
* var index = y * frame.Width + x;
* var depth = _depthPixels[index].Depth;
*
* if (depth > 600 && depth < 900)
* {
* PixelMaker.SetPixel(x, y, Colors.Gold);
* }
* else
* {
* PixelMaker.SetPixel(x, y, Colors.Transparent);
* }
* }
* }*/
//PixelMaker.Draw();
}
}
/// <summary>
/// Starts face tracking from Kinect input data. Track() detects a face
/// based on the passed parameters, then identifies characteristic
/// points and begins tracking. The first call to this API is more
/// expensive, but if the tracking succeeds then subsequent calls use
/// the tracking information generated from first call and is faster,
/// until a tracking failure happens.
/// </summary>
/// <param name="colorImageFormat">
/// format of the colorImage array
/// </param>
/// <param name="colorImage">
/// Input color image frame retrieved from Kinect sensor
/// </param>
/// <param name="depthImageFormat">
/// format of the depthImage array
/// </param>
/// <param name="depthImage">
/// Input depth image frame retrieved from Kinect sensor
/// </param>
/// <returns>
/// Returns computed face tracking results for this image frame
/// </returns>
public FaceTrackFrame Track(
ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage)
{
return(this.Track(colorImageFormat, colorImage, depthImageFormat, depthImage, null, Rect.Empty));
}
private bool CheckFace(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return(false);
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
//getting the Animation Unit Coefficients
this.AUs = frame.GetAnimationUnitCoefficients();
var jawLowerer = AUs[AnimationUnit.JawLower];
var browLower = AUs[AnimationUnit.BrowLower];
var browRaiser = AUs[AnimationUnit.BrowRaiser];
var lipDepressor = AUs[AnimationUnit.LipCornerDepressor];
var lipRaiser = AUs[AnimationUnit.LipRaiser];
var lipStretcher = AUs[AnimationUnit.LipStretcher];
//set up file for output
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt"))
{
file.WriteLine("FaceTrack Data, started recording at " + DateTime.Now.ToString("HH:mm:ss tt"));
}
//here is the algorithm to test different facial features
//BrowLower is messed up if you wear glasses, works if you don't wear 'em
string state = "";
//surprised
if ((jawLowerer < 0.25 || jawLowerer > 0.25) && browLower < 0)
{
state = "surprised";
}
//smiling
if (lipStretcher > 0.4 || lipDepressor < 0)
{
state = "smiling";
}
//sad
if (browRaiser < 0 && lipDepressor > 0)
{
state = "sad";
}
//angry
if ((browLower > 0 && (jawLowerer > 0.25 || jawLowerer < -0.25)) ||
(browLower > 0 && lipDepressor > 0))
{
state = "angry";
}
//System.Diagnostics.Debug.WriteLine(browLower);
this.facePoints = frame.GetProjected3DShape();
if (states[currentState] == state)
{
Trace.WriteLine("Yo!");
return(true);
}
}
}
return(false);
}
public IEnumerable <WeightedRect> DetectFaces(ColorImageFormat colorImageFormat,
byte[] colorImage,
DepthImageFormat depthImageFormat,
short[] depthImage, Rect roi)
{
if (this.operationMode != OperationMode.Kinect)
{
throw new InvalidOperationException(
"Cannot use Track with Kinect input types when face tracker is initialized for tracking videos/images");
}
if (colorImage == null)
{
throw new ArgumentNullException("colorImage");
}
if (depthImage == null)
{
throw new ArgumentNullException("depthImage");
}
if (colorImageFormat != this.initializationColorImageFormat)
{
throw new InvalidOperationException("Color image frame format different from initialization");
}
if (depthImageFormat != this.initializationDepthImageFormat)
{
throw new InvalidOperationException("Depth image frame format different from initialization");
}
if (colorImage.Length != this.videoCameraConfig.FrameBufferLength)
{
throw new ArgumentOutOfRangeException("colorImage", "Color image data size is needs to match initialization configuration.");
}
if (depthImage.Length != this.depthCameraConfig.FrameBufferLength)
{
throw new ArgumentOutOfRangeException("depthImage", "Depth image data size is needs to match initialization configuration.");
}
this.copyStopwatch.Start();
this.colorFaceTrackingImage.CopyFrom(colorImage);
this.depthFaceTrackingImage.CopyFrom(depthImage);
this.copyStopwatch.Stop();
var sensorData = new SensorData(this.colorFaceTrackingImage, this.depthFaceTrackingImage, DefaultZoomFactor, Point.Empty);
FaceTrackingSensorData faceTrackSensorData = sensorData.FaceTrackingSensorData;
int hr;
uint count = 4;
WeightedRect[] rects = new WeightedRect[count];
GCHandle handle = GCHandle.Alloc(rects, GCHandleType.Pinned);
try
{
IntPtr rectsPtr = handle.AddrOfPinnedObject();
hr = this.faceTrackerInteropPtr.DetectFaces(ref faceTrackSensorData, ref roi, rectsPtr, ref count);
}
finally
{
if (handle.IsAllocated)
{
handle.Free();
}
}
this.trackSucceeded = hr == (int)ErrorCode.Success;
return(rects.Take((int)count));
}
private void OnAllFramesReady(object sender, Microsoft.Kinect.AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.SkeletonData == null || this.SkeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.SkeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.SkeletonData);
Skeleton activeSkeleton = null;
activeSkeleton = (from skel in this.SkeletonData where skel.TrackingState == SkeletonTrackingState.Tracked select skel).FirstOrDefault();
//Idea: Separate Eye-Parts of Color Image
//Use learning Algorithm for right and left eye
//Detect blink on separated parts of color Image
//colorImage is one dimensional array with 640 x 480 x 4 (RGBA) values
if (activeSkeleton != null)
{
FaceTrackFrame currentFaceFrame = faceTracker.Track(ColorImageFormat.RgbResolution640x480Fps30, colorImage, depthImageFormat, depthImage, activeSkeleton);
float browRaiserValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowRaiser];
float browLowererValue = currentFaceFrame.GetAnimationUnitCoefficients()[AnimationUnit.BrowLower];
tbBrowLowerer.Text = browLowererValue.ToString();
tbBrowRaiser.Text = browRaiserValue.ToString();
//Get relevant Points for blink detection
//Left eye
int minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].X);
int minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveOneFourthLeftEyelid].Y);
int maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].X);
int maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.BelowThreeFourthLeftEyelid].Y);
Bitmap leftEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, false);
pbLeftEye.Image = leftEye;
//Right eye
minX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].X);
minY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.AboveThreeFourthRightEyelid].Y);
maxX = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].X);
maxY = (int)Math.Round(currentFaceFrame.GetProjected3DShape()[FeaturePoint.OneFourthBottomRightEyelid].Y);
Bitmap rightEye = EyeExtract(colorImageFrame, currentFaceFrame, minX, minY, maxX, maxY, true);
pbRightEye.Image = rightEye;
//Wende Kantenfilter auf die beiden Augen an.
double dxRight;
double dyRight;
double dxLeft;
double dyLeft;
if (rightEye != null && leftEye != null)
{
Bitmap edgePicRight = Convolution(ConvertGrey(rightEye), true, out dxRight, out dyRight);
Bitmap edgePicLeft = Convolution(ConvertGrey(leftEye), false, out dxLeft, out dyLeft);
//If Face is rotated, move Mouse
if (headRotationHistory.Count > filterLength && currentFaceFrame.TrackSuccessful)
{
int x = 0;
int y = 0;
//Method 1: Ohne Glättung
//ScaleXY(currentFaceFrame.Rotation, out x, out y);
//MouseControl.Move(x, y);
////Method 2: Glättung über die letzten x Bilder:
//int i = 0;
//Vector3DF rotationMedium = new Vector3DF();
//while (i < 10 && headRotationHistory.Count - 1 > i)
//{
// i++;
// rotationMedium.X += headRotationHistory[headRotationHistory.Count - 1 - i].X;
// rotationMedium.Y += headRotationHistory[headRotationHistory.Count - 1 - i].Y;
//}
//rotationMedium.X = rotationMedium.X / i;
//rotationMedium.Y = rotationMedium.Y / i;
//ScaleXY(rotationMedium, out x, out y);
//MouseControl.Move(x, y);
//Method 3: Gauß-Filter: Gewichte die letzten Bilder stärker.
Vector3DF rotationMedium = new Vector3DF();
rotationMedium.X = currentFaceFrame.Rotation.X * gaussFilter[0];
rotationMedium.Y = currentFaceFrame.Rotation.Y * gaussFilter[0];
int i = 0;
while (i < filterLength - 1)
{
i++;
rotationMedium.X += (headRotationHistory[headRotationHistory.Count - 1 - i].X * gaussFilter[i]);
rotationMedium.Y += (headRotationHistory[headRotationHistory.Count - 1 - i].Y * gaussFilter[i]);
}
rotationMedium.X = (float)(rotationMedium.X / gaussFactor);
rotationMedium.Y = (float)(rotationMedium.Y / gaussFactor);
ScaleXY(rotationMedium, out x, out y);
MouseControl.Move(x, y);
//Method 4: Quadratische Glättung
//double deltaX = ((-currentFaceFrame.Rotation.Y) - (-headRotationHistory.Last().Y));
//double deltaY = ((-currentFaceFrame.Rotation.X) - (-headRotationHistory.Last().X));
//if (deltaX < 0)
// deltaX = -Math.Pow(deltaX, 2) * 4;
//else
// deltaX = Math.Pow(deltaX, 2) * 4;
//if (deltaY < 0)
// deltaY = -Math.Pow(deltaY, 2) * 5;
//else
// deltaY = Math.Pow(deltaY, 2) * 5;
//MouseControl.DeltaMove((int)Math.Round(deltaX, 0), (int)Math.Round(deltaY));
}
headRotationHistory.Add(currentFaceFrame.Rotation);
if (headRotationHistory.Count >= 100)
{
headRotationHistory.RemoveAt(0);
}
}
}
}
catch (Exception e)
{
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
public void ProcessFrame(KinectSensor sensor, byte[] colorImage, ColorImageFormat colorImageFormat, DepthImageFrame depthFrame, short[] depthImage, DepthImageFormat depthImageFormat, Skeleton[] skeletonData, SkeletonFrame skeletonFrame)
{
//Console.WriteLine("N: ---------");
coordinates.Clear();
int detectedFace = 0;
int trackedSkeletonsCount = 0;
int playerIndex = -1;
for (int i = 0; i < skeletonData.Length; i++)
//foreach (Skeleton skeleton in skeletonData)
{
Skeleton skeleton = skeletonData[i];
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
DepthImagePoint depthPoint = depthFrame.MapFromSkeletonPoint(skeleton.Joints[JointType.Head].Position);
ColorImagePoint colorPoint = depthFrame.MapToColorImagePoint(depthPoint.X, depthPoint.Y, colorImageFormat);
Coordinates2D c = new Coordinates2D();
playerIndex = i + 1;
c.X = colorPoint.X;
c.Y = colorPoint.Y;
c.Width = 0;
c.Height = 0;
c.LeftEyeX = 0;
c.LeftEyeY = 0;
c.RightEyeX = 0;
c.RightEyeY = 0;
c.PlayerIndex = playerIndex;
trackedSkeletonsCount++;
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (!scannedIdentities.Contains(skeleton.TrackingId) &&
detectedFace < 1 &&
trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
detectedFace++;
scannedIdentities.Add(skeleton.TrackingId);
skeletonFaceTracker.OnFrameReady(sensor, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
Coordinates2D?realCoords = skeletonFaceTracker.GetFaceCoordinates();
if (realCoords.HasValue)
{
c = realCoords.Value;
c.PlayerIndex = playerIndex;
}
}
c.TrackingId = skeleton.TrackingId;
coordinates.Add(c);
}
}
if (scannedIdentities.Count > 0 && scannedIdentities.Count >= trackedSkeletonsCount)
{
scannedIdentities.Clear();
//Console.WriteLine("Clearing");
}
RemoveOldTrackers(skeletonFrame.FrameNumber);
//if (coordinates.Count > 0)
{
int[] identities = new int[coordinates.Count];
// stopwatch.Reset();
// stopwatch.Start();
double[] distances = new double[coordinates.Count * 8];
this.
ProcessImage(colorImage, GetWidth(colorImageFormat), GetHeight(colorImageFormat), depthImage, 640, 480, coordinates.ToArray(), identities, distances);
// stopwatch.Stop();
// foreach (int i in identities)
// {
// Console.WriteLine("Recognized: {0} (in {1} millis - {2} ticks)", i, stopwatch.ElapsedMilliseconds, stopwatch.ElapsedTicks);
// }
}
}
/// <summary>
/// Get the size for the given color image format.
/// </summary>
/// <param name="format">The color image format.</param>
/// <returns>The width and height of the given image.</returns>
private static Size GetColorImageSize(ColorImageFormat format)
{
try
{
var q = from item in BackgroundRemovalResolutions
where item.Key == format
select item.Value;
return q.Single();
}
catch (InvalidOperationException)
{
throw new ArgumentException(Resources.UnsupportedColorFormat, "format");
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked
|| skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
private void Kinect_AllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
// TODO look into using the Timestamp on each frame
var time = pipeline.GetCurrentTime();
var sharedColorImage = ImagePool.GetOrCreate(colorImageFrame.Width, colorImageFrame.Height, Imaging.PixelFormat.BGRX_32bpp);
var sharedDepthImage = ImagePool.GetOrCreate(depthImageFrame.Width, depthImageFrame.Height, Imaging.PixelFormat.Gray_16bpp);
colorImageFrame.CopyPixelDataTo(sharedColorImage.Resource.ImageData, (colorImageFrame.Width * colorImageFrame.Height * 4));
this.ColorImage.Post(sharedColorImage, time);
//depthImageFrame.CopyPixelDataTo(sharedDepthImage.Resource.ImageData, (depthImageFrame.Width * depthImageFrame.Height * 2));
depthImageFrame.CopyPixelDataTo(sharedDepthImage.Resource.ImageData, depthImageFrame.PixelDataLength);
this.DepthImage.Post(sharedDepthImage, time);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
this.Skeletons.Post(this.skeletonData.ToList(), time);
}
catch
{
// TODO catch a cold
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
Trace.WriteLine("allframesready");
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
Trace.WriteLine("try start");
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
Trace.WriteLine("return from frameready");
return;
}
Trace.WriteLine("wayaround null");
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
Trace.WriteLine("depth image fromat no work");
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
Trace.WriteLine("color image fromat no work");
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
Trace.WriteLine("starting copying of data");
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
Trace.WriteLine("end copying of data");
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
Trace.WriteLine("inside foreach loop");
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
Trace.WriteLine("iftracked");
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
Trace.WriteLine("add skeleton tracking");
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
Trace.WriteLine("before");
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton, this.activityWindow);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
/// <summary>
/// Set the background removed color frame format.
/// </summary>
/// <param name="format">
/// The given color image format.
/// </param>
/// <param name="forceEnable">
/// Streams should be enabled even if new color image format is the same as the old one.
/// This is useful for the initial enabling of the stream.
/// </param>
private void UpdateBackgroundRemovalFrameFormat(ColorImageFormat format, bool forceEnable)
{
if (!forceEnable && (format == this.colorImageFormat))
{
// No work to do
return;
}
if (this.sensor != null)
{
try
{
this.sensor.ColorStream.Enable(format);
this.backgroundRemovalStream.Enable(format, DepthImageFormat.Resolution640x480Fps30);
}
catch (InvalidOperationException)
{
// KinectSensor might enter an invalid state while enabling/disabling streams or stream features.
// E.g.: sensor might be abruptly unplugged.
}
}
// Update the image format property if the action succeeded.
this.colorImageFormat = format;
}
private void AllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for changes in any of the data this function is receiving
// and reset things appropriately.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.DestroyFaceTracker();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.DestroyFaceTracker();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
this.colorImageWritableBitmap = null;
this.ColorImage.Source = null;
this.theMaterial.Brush = null;
}
if (this.skeletonData != null && this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = null;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
if (this.colorImageWritableBitmap == null)
{
this.colorImageWritableBitmap = new WriteableBitmap(
colorImageFrame.Width, colorImageFrame.Height, 96, 96, PixelFormats.Bgr32, null);
this.ColorImage.Source = this.colorImageWritableBitmap;
this.theMaterial.Brush = new ImageBrush(this.colorImageWritableBitmap)
{
ViewportUnits = BrushMappingMode.Absolute
};
}
if (this.skeletonData == null)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
// Copy data received in this event to our buffers.
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
this.colorImageWritableBitmap.WritePixels(
new Int32Rect(0, 0, colorImageFrame.Width, colorImageFrame.Height),
this.colorImage,
colorImageFrame.Width * Bgr32BytesPerPixel,
0);
// Find a skeleton to track.
// First see if our old one is good.
// When a skeleton is in PositionOnly tracking state, don't pick a new one
// as it may become fully tracked again.
Skeleton skeletonOfInterest =
this.skeletonData.FirstOrDefault(
skeleton =>
skeleton.TrackingId == this.trackingId &&
skeleton.TrackingState != SkeletonTrackingState.NotTracked);
if (skeletonOfInterest == null)
{
// Old one wasn't around. Find any skeleton that is being tracked and use it.
skeletonOfInterest =
this.skeletonData.FirstOrDefault(
skeleton => skeleton.TrackingState == SkeletonTrackingState.Tracked);
if (skeletonOfInterest != null)
{
// This may be a different person so reset the tracker which
// could have tuned itself to the previous person.
if (this.faceTracker != null)
{
this.faceTracker.ResetTracking();
}
this.trackingId = skeletonOfInterest.TrackingId;
}
}
if (skeletonOfInterest != null && skeletonOfInterest.TrackingState == SkeletonTrackingState.Tracked)
{
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(this.Kinect);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame faceTrackFrame = this.faceTracker.Track(
this.colorImageFormat,
this.colorImage,
this.depthImageFormat,
this.depthImage,
skeletonOfInterest);
if (faceTrackFrame.TrackSuccessful)
{
if (!visited)
{
visited = true;
//counter.Text = "60 seconds";
aTimer.Interval = 1000;
aTimer.Tick += new EventHandler(aTimer_Tick);
aTimer.Start();
}
if (saveModel)
{
saveDepthImagebmp(depthImageFrame);
saveColorImage(colorImageFrame.Width, colorImageFrame.Height, (colorImageFrame.Width * Bgr32BytesPerPixel));
saveFaceModel();
}
}
}
}
else
{
this.trackingId = -1;
}
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
/// <summary>
/// Updates the face tracking information for this skeleton
/// </summary>
internal void OnFrameReady(KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)
{
this.skeletonTrackingState = skeletonOfInterest.TrackingState;
if (this.skeletonTrackingState != SkeletonTrackingState.Tracked)
{
// nothing to do with an untracked skeleton.
return;
}
if (this.faceTracker == null)
{
try
{
this.faceTracker = new FaceTracker(kinectSensor);
}
catch (InvalidOperationException)
{
// During some shutdown scenarios the FaceTracker
// is unable to be instantiated. Catch that exception
// and don't track a face.
Debug.WriteLine("AllFramesReady - creating a new FaceTracker threw an InvalidOperationException");
this.faceTracker = null;
}
}
if (this.faceTracker != null)
{
FaceTrackFrame frame = this.faceTracker.Track(
colorImageFormat, colorImage, depthImageFormat, depthImage, skeletonOfInterest);
this.lastFaceTrackSucceeded = frame.TrackSuccessful;
if (this.lastFaceTrackSucceeded)
{
if (faceTriangles == null)
{
// only need to get this once. It doesn't change.
faceTriangles = frame.GetTriangles();
}
//getting the Animation Unit Coefficients
this.AUs = frame.GetAnimationUnitCoefficients();
var jawLowerer = AUs[AnimationUnit.JawLower];
var browLower = AUs[AnimationUnit.BrowLower];
var browRaiser = AUs[AnimationUnit.BrowRaiser];
var lipDepressor = AUs[AnimationUnit.LipCornerDepressor];
var lipRaiser = AUs[AnimationUnit.LipRaiser];
var lipStretcher = AUs[AnimationUnit.LipStretcher];
//set up file for output
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt"))
{
file.WriteLine("FaceTrack Data, started recording at " + DateTime.Now.ToString("HH:mm:ss tt"));
}
//here is the algorithm to test different facial features
//BrowLower is messed up if you wear glasses, works if you don't wear 'em
//surprised
if ((jawLowerer < 0.25 || jawLowerer > 0.25) && browLower < 0)
{
System.Diagnostics.Debug.WriteLine("surprised");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": surprised");
file.WriteLine("JawLowerer: " + jawLowerer);
file.WriteLine("BrowLowerer: " + browLower);
}
}
//smiling
if (lipStretcher > 0.4 || lipDepressor<0)
{
System.Diagnostics.Debug.WriteLine("Smiling");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": smiling");
file.WriteLine("LipStretcher: " + lipStretcher);
}
}
//kissing face
if (lipStretcher < -0.75)
{
System.Diagnostics.Debug.WriteLine("kissing face");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": kissing face");
file.WriteLine("LipStretcher: " + lipStretcher);
}
}
//sad
if (browRaiser < 0 && lipDepressor>0)
{
System.Diagnostics.Debug.WriteLine("sad");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": sad");
file.WriteLine("LipCornerDepressor: " + lipDepressor);
file.WriteLine("OuterBrowRaiser: " + browRaiser);
}
}
//angry
if ((browLower > 0 && (jawLowerer > 0.25 || jawLowerer < -0.25)) ||
(browLower > 0 && lipDepressor > 0))
{
System.Diagnostics.Debug.WriteLine("angry");
using (System.IO.StreamWriter file = new System.IO.StreamWriter
(@"C:\Users\Public\data.txt", true))
{
file.WriteLine(DateTime.Now.ToString("HH:mm:ss tt") + ": angry");
file.WriteLine("LipCornerDepressor: " + lipDepressor);
file.WriteLine("BrowLowerer: " + browLower);
file.WriteLine("JawLowerer: " + jawLowerer);
}
}
//System.Diagnostics.Debug.WriteLine(browLower);
this.facePoints = frame.GetProjected3DShape();
}
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.resetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.resetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
try
{
this.trackedSkeleton.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
}
catch (NullReferenceException)
{
//se si perder il tracking del viso si evita un crash
}
this.trackedSkeleton.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
