/**
* Update the View
*/
private void update()
{
while (model.SenseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) // Got an image?
{
// <magic>
PXCMCapture.Sample sample = model.SenseManager.QueryFaceSample();
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
model.FaceData = model.Face.CreateOutput();
model.FaceData.Update(); // props to Tanja
model.FaceAktuell = model.FaceData.QueryFaceByIndex(0);
if (model.FaceAktuell != null)
{
model.Edata = model.FaceAktuell.QueryExpressions();
}
model.HandData = model.Hand.CreateOutput();
model.HandData.Update();
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
Graphics bitmapGraphics = Graphics.FromImage(colorBitmap);
model.Modules.ForEach(delegate(RSModule mod)
{
mod.Work(bitmapGraphics);
});
// update PictureBox
pb.Image = colorBitmap;
model.SenseManager.ReleaseFrame();
model.FaceData.Dispose(); // DONE!
model.HandData.Dispose();
model.Edata = null;
}
}
/**
* Update the View
*/
private void update()
{
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) // Got an image?
{
// Console.WriteLine("Update");
// <magic>
PXCMCapture.Sample sample = senseManager.QueryFaceSample();
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB24, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
Graphics bitmapGraphics = Graphics.FromImage(colorBitmap);
modules.ForEach(delegate(RSModule mod)
{
mod.Work(bitmapGraphics);
});
// </magic>
// save to hard drive (careful!) - will be stored in project folder/bin/debug
if (storeImages)
{
colorBitmap.Save("cap" + capNum++ + ".png");
}
// update PictureBox
pb.Image = colorBitmap;
senseManager.ReleaseFrame();
}
}
pxcmStatus newIRFrame(int mid, PXCMCapture.Sample sample)
{
if (sample.ir != null && irImageForm != null && !irImageForm.IsDisposed)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
sample.ir.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
irImageForm.Image = data.ToBitmap(0, sample.ir.info.width, sample.ir.info.height);
sample.ir.ReleaseAccess(data);
}
return(pxcmStatus.PXCM_STATUS_NO_ERROR);
}
pxcmStatus newHandFrame(PXCMHandModule hand)
{
if (hand != null)
{
PXCMHandData handData = hand.CreateOutput();
handData.Update();
PXCMHandData.IHand iHandDataLeft = null, iHandDataRight = null;
PXCMHandData.JointData jointData = null;
PXCMImage image = null;
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_LEFT_HANDS, 0, out iHandDataLeft);
handData.QueryHandData(PXCMHandData.AccessOrderType.ACCESS_ORDER_RIGHT_HANDS, 0, out iHandDataRight);
if (handForm != null && !handForm.IsDisposed)
{
this.handForm.HandCount = handData.QueryNumberOfHands();
if (iHandDataLeft != null)
{
iHandDataLeft.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.LeftHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
if (iHandDataRight != null)
{
iHandDataRight.QuerySegmentationImage(out image);
if (image != null)
{
PXCMImage.ImageData data = new PXCMImage.ImageData();
image.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out data);
handForm.RightHand = data.ToBitmap(0, image.info.width, image.info.height);
image.ReleaseAccess(data);
}
}
}
if (iHandDataLeft != null)
{
if (jointData == null)
{
iHandDataLeft.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (iHandDataRight != null)
{
if (jointData == null)
{
iHandDataRight.QueryTrackedJoint(PXCMHandData.JointType.JOINT_INDEX_TIP, out jointData);
}
}
if (jointData != null && canTrack.Checked)
{
Cursor.Position = new System.Drawing.Point(
(int)((640.0f - jointData.positionImage.x) * Screen.PrimaryScreen.Bounds.Width / 640.0f),
(int)(jointData.positionImage.y * Screen.PrimaryScreen.Bounds.Height / 480.0f));
PXCMHandData.GestureData gestureData = null;
if (handData.IsGestureFired("two_fingers_pinch_open", out gestureData))
{
Program.DoMouseClick();
}
Console.WriteLine("Z Position: " + jointData.positionWorld.z);
}
handData.Dispose();
}
return(pxcmStatus.PXCM_STATUS_NO_ERROR);
}
/**
* Update the View and modules
*/
private void update()
{
Stopwatch stopwatch = new Stopwatch();
while (true)
{
if (model.SenseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) // Dauert manchmal voll lange ...
{
debug_y = 0;
// <magic>
PXCMCapture.Sample sample = model.SenseManager.QueryFaceSample();
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData);
model.FaceData = model.Face.CreateOutput();
model.FaceData.Update();
model.FaceAktuell = model.FaceData.QueryFaceByIndex(0);
if (model.FaceAktuell != null)
{
uiSlide = true;
PXCMFaceData.PoseData pose = model.FaceAktuell.QueryPose();
if (pose != null)
{
pose.QueryPoseAngles(out model.currentPose);
}
model.Lp = model.FaceAktuell.QueryLandmarks();
if (model.NullFace == null)
{
if (model.Lp != null)
{
PXCMFaceData.LandmarkPoint[] aPoints;
model.Lp.QueryPoints(out aPoints);
model.NullFace = aPoints;
}
}
}
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
Graphics bitmapGraphics = Graphics.FromImage(colorBitmap);
if (resetModules)
{
model.Modules.ForEach(delegate(RSModule mod)
{
mod.Reset();
});
resetModules = false;
}
if (testMode)
{
if (outputEnabled)
{
bitmapGraphics.FillRectangle(model.DefaultBGBrush, new Rectangle(0, 0, model.Width, model.Height));
}
if (!imageEnabled)
{
bitmapGraphics.FillRectangle(model.OpaqueBGBrush, new Rectangle(0, 0, model.Width, model.Height));
}
bitmapGraphics.DrawString("pose: " + model.CurrentPoseDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
Debug_Y += 25;
}
else
{
using (Graphics gr = Graphics.FromImage(colorBitmap))
{
if (model.calibrationProgress == 100)
{
angerMonitor.targetValue = (int)model.Emotions[Model.Emotion.ANGER];
fearMonitor.targetValue = (int)model.Emotions[Model.Emotion.FEAR];
disgustMonitor.targetValue = (int)model.Emotions[Model.Emotion.DISGUST];
surpriseMonitor.targetValue = (int)model.Emotions[Model.Emotion.SURPRISE];
joyMonitor.targetValue = (int)model.Emotions[Model.Emotion.JOY];
sadMonitor.targetValue = (int)model.Emotions[Model.Emotion.SADNESS];
contemptMonitor.targetValue = (int)model.Emotions[Model.Emotion.CONTEMPT];
angerMonitor.Step();
fearMonitor.Step();
disgustMonitor.Step();
surpriseMonitor.Step();
joyMonitor.Step();
sadMonitor.Step();
contemptMonitor.Step();
}
if (!testMode)
{
gr.DrawImage(windowBitmap, xP - 90, yP - 150);
}
FriggnAweseomeGraphix.DrawMEMontior(gr, angerMonitor);
FriggnAweseomeGraphix.DrawMEMontior(gr, sadMonitor);
FriggnAweseomeGraphix.DrawMEMontior(gr, fearMonitor);
FriggnAweseomeGraphix.DrawMEMontior(gr, surpriseMonitor);
FriggnAweseomeGraphix.DrawMEMontior(gr, contemptMonitor);
FriggnAweseomeGraphix.DrawMEMontior(gr, disgustMonitor);
FriggnAweseomeGraphix.DrawMEMontior(gr, joyMonitor);
bitmapGraphics.DrawString("Subject #" + subject, FriggnAweseomeGraphix.majorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP, yP - 75);
bitmapGraphics.DrawLine(linePen, xP + 10, yP - 20, xP + 800, yP - 20);
bitmapGraphics.DrawString("Pose: " + (int)model.CurrentPoseDiff, FriggnAweseomeGraphix.minorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP + 550, yP - 55);
FriggnAweseomeGraphix.MEMonitor calibMonitor = new FriggnAweseomeGraphix.MEMonitor("", "", 1150, 580 - calibRadius, calibRadius, 20);
calibMonitor.showPercent = false;
if (model.calibrationProgress != 100 && model.CurrentFace != null)
{
PXCMFaceData.LandmarkPoint mPoint = model.CurrentFace[29];
int rad = 2;
if (mPoint.image.x > 1200)
{
calibMonitor.y = (int)mPoint.image.y - calibRadius;
calibMonitor.x = (int)mPoint.image.x - calibRadius;
}
calibMonitor.showPercent = false;
calibMonitor.currentValue = (int)model.calibrationProgress;
int sMark = (int)(model.calibrationProgress * 0.69);
int tMark = sMark + 1;
PXCMFaceData.LandmarkPoint sPoint = model.CurrentFace[sMark];
PXCMFaceData.LandmarkPoint tPoint = model.CurrentFace[tMark];
SolidBrush sb = new SolidBrush(FriggnAweseomeGraphix.fgColor);
for (int i = 0; i < sMark - 1; i++)
{
PXCMFaceData.LandmarkPoint iPoint = model.CurrentFace[i];
gr.FillEllipse(sb, new Rectangle((int)iPoint.image.x - rad, (int)iPoint.image.y - rad, rad * 2, rad * 2));
}
gr.FillEllipse(sb, new Rectangle((int)sPoint.image.x - rad, (int)sPoint.image.y - rad, rad * 4, rad * 4));
FriggnAweseomeGraphix.DrawMEMontior(gr, calibMonitor);
}
else
{
calibMonitor.currentValue = 0;
}
if (model.CurrentPoseDiff > model.PoseMax && model.calibrationProgress == 100)
{
warningPopupX += (warningPopupSlideX - warningPopupX) / 3;
gr.FillRectangle(new SolidBrush(Color.FromArgb(100, 255, 255, 255)), new Rectangle(warningX, warningY, warningWidth, warningHeight));
gr.DrawImage(smallWindowBitmap, warningPopupX, 900);
}
else
{
warningPopupX = 2500;
}
}
}
if (model.FaceData.QueryNumberOfDetectedFaces() > 0 && model.CurrentFace != null)
{
model.Modules.ForEach(delegate(RSModule mod)
{
mod.Work(bitmapGraphics);
if (outputEnabled && mod.output != "")
{
bitmapGraphics.DrawString(mod.output, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
Debug_Y += 25; // new row
}
});
}
double pitchDiff = Math.Abs(model.currentPose.pitch - model.NullPose.pitch);
double rollDiff = Math.Abs(model.currentPose.roll - model.NullPose.roll);
double yawDiff = Math.Abs(model.currentPose.yaw - model.NullPose.yaw);
model.CurrentPoseDiff = pitchDiff + rollDiff + yawDiff;
model.CurrentRollDiff = rollDiff;
model.CurrentPitchDiff = pitchDiff;
model.CurrentYawDiff = yawDiff;
// Update PictureBox
if (testMode)
{
pb.Image = colorBitmap;
}
else
{
pb.Image = colorBitmap; // uiBitmap.Clone(new Rectangle(0, 0, uiBitmap.Width, uiBitmap.Height), uiBitmap.PixelFormat);
}
if (PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL != model.SenseManager.captureManager.device.QueryMirrorMode())
{
model.SenseManager.captureManager.device.SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL); //mirror
}
model.SenseManager.ReleaseFrame();
model.FaceData.Dispose(); // DONE!
sample.color.ReleaseAccess(colorData);
if (xP < targetX && uiSlide)
{
xP += (targetX - xP) / 8;
warningX = xP - 70;
warningSymbolX = (warningX + warningWidth) / 2 - 728 / 2;
angerMonitor.x = xP;
joyMonitor.x = xP;
fearMonitor.x = xP;
contemptMonitor.x = xP;
sadMonitor.x = xP + xgap;
disgustMonitor.x = xP + xgap;
surpriseMonitor.x = xP + xgap;
}
}
}
}
/**
* Update the View
*/
private void update()
{
Stopwatch stopwatch = new Stopwatch();
while (true)
{
if (model.SenseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) // Dauert manchmal voll lange ...
{
debug_y = 0;
// <magic>
PXCMCapture.Sample sample = model.SenseManager.QueryFaceSample();
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData);
model.FaceData = model.Face.CreateOutput();
model.FaceData.Update();
model.FaceAktuell = model.FaceData.QueryFaceByIndex(0);
if (model.FaceAktuell != null)
{
uiSlide = true;
PXCMFaceData.PoseData pose = model.FaceAktuell.QueryPose();
if (pose != null)
{
pose.QueryPoseAngles(out model.currentPose);
}
model.Lp = model.FaceAktuell.QueryLandmarks();
if (model.NullFace == null)
{
if (model.Lp != null)
{
PXCMFaceData.LandmarkPoint[] aPoints;
model.Lp.QueryPoints(out aPoints);
model.NullFace = aPoints;
}
}
}
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
Graphics bitmapGraphics = Graphics.FromImage(colorBitmap);
if (resetModules)
{
model.Modules.ForEach(delegate(RSModule mod)
{
mod.reset();
});
resetModules = false;
}
if (testMode)
{
if (outputEnabled)
{
bitmapGraphics.FillRectangle(model.DefaultBGBrush, new Rectangle(0, 0, model.Width, model.Height));
}
if (!imageEnabled)
{
bitmapGraphics.FillRectangle(model.OpaqueBGBrush, new Rectangle(0, 0, model.Width, model.Height));
}
bitmapGraphics.DrawString("pose: " + model.CurrentPoseDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
Debug_Y += 25;
}
else
{
// uiBitmap = new Bitmap(blurWidth, blurHeight, PixelFormat.Format32bppArgb);
// using (Graphics gr = Graphics.FromImage(uiBitmap))
// {
// gr.DrawImage(colorBitmap, 0, 0);
// }
// BitmapData sourceData = colorBitmap.LockBits(new Rectangle(xPos - 10, yPos - 10, blurWidth + 20, blurHeight + 20), ImageLockMode.ReadOnly, colorBitmap.PixelFormat);
//BitmapData uiData = uiBitmap.LockBits(new Rectangle(0, 0, blurWidth, blurHeight), ImageLockMode.WriteOnly, uiBitmap.PixelFormat);
//if (blur) FriggnAweseomeGraphix.sonic_blur(colorBitmap, uiBitmap, 0, 0, blurWidth, blurHeight, 1, 4, sourceData, uiData);
// uiBitmap.UnlockBits(uiData);
// colorBitmap.UnlockBits(sourceData);
using (Graphics gr = Graphics.FromImage(colorBitmap))
{
if (model.calibrationProgress == 100)
{
angerMonitor.targetValue = (int)model.Emotions["Anger"];
fearMonitor.targetValue = (int)model.Emotions["Fear"];
disgustMonitor.targetValue = (int)model.Emotions["Disgust"];
surpriseMonitor.targetValue = (int)model.Emotions["Surprise"];
joyMonitor.targetValue = (int)model.Emotions["Joy"];
sadMonitor.targetValue = (int)model.Emotions["Sadness"];
angerMonitor.step();
fearMonitor.step();
disgustMonitor.step();
surpriseMonitor.step();
joyMonitor.step();
sadMonitor.step();
}
// if (false) gr.DrawImage(uiBitmap, xPos, yPos);
gr.DrawImage(windowBitmap, xP - 90, yP - 150);
FriggnAweseomeGraphix.drawMEMontior(gr, angerMonitor);
FriggnAweseomeGraphix.drawMEMontior(gr, sadMonitor);
FriggnAweseomeGraphix.drawMEMontior(gr, fearMonitor);
FriggnAweseomeGraphix.drawMEMontior(gr, surpriseMonitor);
FriggnAweseomeGraphix.drawMEMontior(gr, contemptMonitor);
FriggnAweseomeGraphix.drawMEMontior(gr, disgustMonitor);
FriggnAweseomeGraphix.drawMEMontior(gr, joyMonitor);
bitmapGraphics.DrawString("Subject #" + subject, FriggnAweseomeGraphix.majorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP, yP - 75);
bitmapGraphics.DrawLine(linePen, xP + 10, yP - 20, xP + 800, yP - 20);
bitmapGraphics.DrawString("Pose: " + (int)model.CurrentPoseDiff, FriggnAweseomeGraphix.minorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP + 550, yP - 55);
FriggnAweseomeGraphix.MEMonitor calibMonitor = new FriggnAweseomeGraphix.MEMonitor("", "", 1150, 580 - calibRadius, calibRadius, 20);
calibMonitor.showPercent = false;
if (model.calibrationProgress != 100 && model.CurrentFace != null)
{
PXCMFaceData.LandmarkPoint mPoint = model.CurrentFace[29];
int rad = 2;
if (mPoint.image.x > 1200)
{
calibMonitor.y = (int)mPoint.image.y - calibRadius;
calibMonitor.x = (int)mPoint.image.x - calibRadius;
}
calibMonitor.showPercent = false;
calibMonitor.currentValue = (int)model.calibrationProgress;
int sMark = (int)(model.calibrationProgress * 0.69);
int tMark = sMark + 1;
Console.WriteLine(sMark + " -> " + tMark);
PXCMFaceData.LandmarkPoint sPoint = model.CurrentFace[sMark];
PXCMFaceData.LandmarkPoint tPoint = model.CurrentFace[tMark];
//FriggnAweseomeGraphix.drawFadingLine(gr, sPoint.image.x, sPoint.image.y, tPoint.image.x, tPoint.image.y);
SolidBrush sb = new SolidBrush(FriggnAweseomeGraphix.fgColor);
for (int i = 0; i < sMark - 1; i++)
{
PXCMFaceData.LandmarkPoint iPoint = model.CurrentFace[i];
gr.FillEllipse(sb, new Rectangle((int)iPoint.image.x - rad, (int)iPoint.image.y - rad, rad * 2, rad * 2));
}
gr.FillEllipse(sb, new Rectangle((int)sPoint.image.x - rad, (int)sPoint.image.y - rad, rad * 4, rad * 4));
FriggnAweseomeGraphix.drawMEMontior(gr, calibMonitor);
}
else
{
calibMonitor.currentValue = 0;
}
if (model.CurrentPoseDiff > model.PoseMax && model.calibrationProgress == 100)
{
warningPopupX += (warningPopupSlideX - warningPopupX) / 3;
gr.FillRectangle(new SolidBrush(Color.FromArgb(100, 255, 255, 255)), new Rectangle(warningX, warningY, warningWidth, warningHeight));
gr.DrawImage(smallWindowBitmap, warningPopupX, 900);
}
else
{
warningPopupX = 2500;
}
}
}
if (model.FaceData.QueryNumberOfDetectedFaces() > 0 && model.CurrentFace != null)
{
model.Modules.ForEach(delegate(RSModule mod)
{
mod.Work(bitmapGraphics);
if (outputEnabled && mod.output != "")
{
bitmapGraphics.DrawString(mod.output, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
Debug_Y += 25; // new row
}
});
}
double pitchDiff = Math.Abs(model.currentPose.pitch - model.NullPose.pitch);
double rollDiff = Math.Abs(model.currentPose.roll - model.NullPose.roll);
double yawDiff = Math.Abs(model.currentPose.yaw - model.NullPose.yaw);
model.CurrentPoseDiff = pitchDiff + rollDiff + yawDiff;
model.CurrentRollDiff = rollDiff;
model.CurrentPitchDiff = pitchDiff;
model.CurrentYawDiff = yawDiff;
/* bitmapGraphics.DrawString("poll: " + pitchDiff + ", roll: " + rollDiff + ", yaw: " + yawDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
* Debug_Y += 25;
* bitmapGraphics.DrawString("all: " + (int)(pitchDiff + rollDiff + yawDiff), model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
* Debug_Y += 25;
* bitmapGraphics.DrawString("pitch: " + pitchDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
* Debug_Y += 25;
* bitmapGraphics.DrawString("yaw: " + yawDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);
* Debug_Y += 25;
* bitmapGraphics.DrawString("roll: " + rollDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);*/
// update PictureBox
if (testMode)
{
pb.Image = colorBitmap;
}
else
{
pb.Image = colorBitmap; // uiBitmap.Clone(new Rectangle(0, 0, uiBitmap.Width, uiBitmap.Height), uiBitmap.PixelFormat);
}
model.SenseManager.ReleaseFrame();
model.FaceData.Dispose(); // DONE!
sample.color.ReleaseAccess(colorData);
if (xP < targetX && uiSlide)
{
xP += (targetX - xP) / 8;
warningX = xP - 70;
warningSymbolX = (warningX + warningWidth) / 2 - 728 / 2;
angerMonitor.x = xP;
joyMonitor.x = xP;
fearMonitor.x = xP;
contemptMonitor.x = xP;
sadMonitor.x = xP + xgap;
disgustMonitor.x = xP + xgap;
surpriseMonitor.x = xP + xgap;
}
}
}
}
