/** * Update the View */ private void update() { Stopwatch stopwatch = new Stopwatch(); while (true) { if (model.SenseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) // Dauert manchmal voll lange ... { debug_y = 0; // <magic> PXCMCapture.Sample sample = model.SenseManager.QueryFaceSample(); sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData); model.FaceData = model.Face.CreateOutput(); model.FaceData.Update(); model.FaceAktuell = model.FaceData.QueryFaceByIndex(0); if (model.FaceAktuell != null) { uiSlide = true; PXCMFaceData.PoseData pose = model.FaceAktuell.QueryPose(); if (pose != null) { pose.QueryPoseAngles(out model.currentPose); } model.Lp = model.FaceAktuell.QueryLandmarks(); if (model.NullFace == null) { if (model.Lp != null) { PXCMFaceData.LandmarkPoint[] aPoints; model.Lp.QueryPoints(out aPoints); model.NullFace = aPoints; } } } colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); Graphics bitmapGraphics = Graphics.FromImage(colorBitmap); if (resetModules) { model.Modules.ForEach(delegate(RSModule mod) { mod.reset(); }); resetModules = false; } if (testMode) { if (outputEnabled) { bitmapGraphics.FillRectangle(model.DefaultBGBrush, new Rectangle(0, 0, model.Width, model.Height)); } if (!imageEnabled) { bitmapGraphics.FillRectangle(model.OpaqueBGBrush, new Rectangle(0, 0, model.Width, model.Height)); } bitmapGraphics.DrawString("pose: " + model.CurrentPoseDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); Debug_Y += 25; } else { // uiBitmap = new Bitmap(blurWidth, blurHeight, PixelFormat.Format32bppArgb); // using (Graphics gr = Graphics.FromImage(uiBitmap)) // { // gr.DrawImage(colorBitmap, 0, 0); // } // BitmapData sourceData = colorBitmap.LockBits(new Rectangle(xPos - 10, yPos - 10, blurWidth + 20, blurHeight + 20), ImageLockMode.ReadOnly, colorBitmap.PixelFormat); //BitmapData uiData = uiBitmap.LockBits(new Rectangle(0, 0, blurWidth, blurHeight), ImageLockMode.WriteOnly, uiBitmap.PixelFormat); //if (blur) FriggnAweseomeGraphix.sonic_blur(colorBitmap, uiBitmap, 0, 0, blurWidth, blurHeight, 1, 4, sourceData, uiData); // uiBitmap.UnlockBits(uiData); // colorBitmap.UnlockBits(sourceData); using (Graphics gr = Graphics.FromImage(colorBitmap)) { if (model.calibrationProgress == 100) { angerMonitor.targetValue = (int)model.Emotions["Anger"]; fearMonitor.targetValue = (int)model.Emotions["Fear"]; disgustMonitor.targetValue = (int)model.Emotions["Disgust"]; surpriseMonitor.targetValue = (int)model.Emotions["Surprise"]; joyMonitor.targetValue = (int)model.Emotions["Joy"]; sadMonitor.targetValue = (int)model.Emotions["Sadness"]; angerMonitor.step(); fearMonitor.step(); disgustMonitor.step(); surpriseMonitor.step(); joyMonitor.step(); sadMonitor.step(); } // if (false) gr.DrawImage(uiBitmap, xPos, yPos); gr.DrawImage(windowBitmap, xP - 90, yP - 150); FriggnAweseomeGraphix.drawMEMontior(gr, angerMonitor); FriggnAweseomeGraphix.drawMEMontior(gr, sadMonitor); FriggnAweseomeGraphix.drawMEMontior(gr, fearMonitor); FriggnAweseomeGraphix.drawMEMontior(gr, surpriseMonitor); FriggnAweseomeGraphix.drawMEMontior(gr, contemptMonitor); FriggnAweseomeGraphix.drawMEMontior(gr, disgustMonitor); FriggnAweseomeGraphix.drawMEMontior(gr, joyMonitor); bitmapGraphics.DrawString("Subject #" + subject, FriggnAweseomeGraphix.majorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP, yP - 75); bitmapGraphics.DrawLine(linePen, xP + 10, yP - 20, xP + 800, yP - 20); bitmapGraphics.DrawString("Pose: " + (int)model.CurrentPoseDiff, FriggnAweseomeGraphix.minorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP + 550, yP - 55); FriggnAweseomeGraphix.MEMonitor calibMonitor = new FriggnAweseomeGraphix.MEMonitor("", "", 1150, 580 - calibRadius, calibRadius, 20); calibMonitor.showPercent = false; if (model.calibrationProgress != 100 && model.CurrentFace != null) { PXCMFaceData.LandmarkPoint mPoint = model.CurrentFace[29]; int rad = 2; if (mPoint.image.x > 1200) { calibMonitor.y = (int)mPoint.image.y - calibRadius; calibMonitor.x = (int)mPoint.image.x - calibRadius; } calibMonitor.showPercent = false; calibMonitor.currentValue = (int)model.calibrationProgress; int sMark = (int)(model.calibrationProgress * 0.69); int tMark = sMark + 1; Console.WriteLine(sMark + " -> " + tMark); PXCMFaceData.LandmarkPoint sPoint = model.CurrentFace[sMark]; PXCMFaceData.LandmarkPoint tPoint = model.CurrentFace[tMark]; //FriggnAweseomeGraphix.drawFadingLine(gr, sPoint.image.x, sPoint.image.y, tPoint.image.x, tPoint.image.y); SolidBrush sb = new SolidBrush(FriggnAweseomeGraphix.fgColor); for (int i = 0; i < sMark - 1; i++) { PXCMFaceData.LandmarkPoint iPoint = model.CurrentFace[i]; gr.FillEllipse(sb, new Rectangle((int)iPoint.image.x - rad, (int)iPoint.image.y - rad, rad * 2, rad * 2)); } gr.FillEllipse(sb, new Rectangle((int)sPoint.image.x - rad, (int)sPoint.image.y - rad, rad * 4, rad * 4)); FriggnAweseomeGraphix.drawMEMontior(gr, calibMonitor); } else { calibMonitor.currentValue = 0; } if (model.CurrentPoseDiff > model.PoseMax && model.calibrationProgress == 100) { warningPopupX += (warningPopupSlideX - warningPopupX) / 3; gr.FillRectangle(new SolidBrush(Color.FromArgb(100, 255, 255, 255)), new Rectangle(warningX, warningY, warningWidth, warningHeight)); gr.DrawImage(smallWindowBitmap, warningPopupX, 900); } else { warningPopupX = 2500; } } } if (model.FaceData.QueryNumberOfDetectedFaces() > 0 && model.CurrentFace != null) { model.Modules.ForEach(delegate(RSModule mod) { mod.Work(bitmapGraphics); if (outputEnabled && mod.output != "") { bitmapGraphics.DrawString(mod.output, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); Debug_Y += 25; // new row } }); } double pitchDiff = Math.Abs(model.currentPose.pitch - model.NullPose.pitch); double rollDiff = Math.Abs(model.currentPose.roll - model.NullPose.roll); double yawDiff = Math.Abs(model.currentPose.yaw - model.NullPose.yaw); model.CurrentPoseDiff = pitchDiff + rollDiff + yawDiff; model.CurrentRollDiff = rollDiff; model.CurrentPitchDiff = pitchDiff; model.CurrentYawDiff = yawDiff; /* bitmapGraphics.DrawString("poll: " + pitchDiff + ", roll: " + rollDiff + ", yaw: " + yawDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); * Debug_Y += 25; * bitmapGraphics.DrawString("all: " + (int)(pitchDiff + rollDiff + yawDiff), model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); * Debug_Y += 25; * bitmapGraphics.DrawString("pitch: " + pitchDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); * Debug_Y += 25; * bitmapGraphics.DrawString("yaw: " + yawDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); * Debug_Y += 25; * bitmapGraphics.DrawString("roll: " + rollDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y);*/ // update PictureBox if (testMode) { pb.Image = colorBitmap; } else { pb.Image = colorBitmap; // uiBitmap.Clone(new Rectangle(0, 0, uiBitmap.Width, uiBitmap.Height), uiBitmap.PixelFormat); } model.SenseManager.ReleaseFrame(); model.FaceData.Dispose(); // DONE! sample.color.ReleaseAccess(colorData); if (xP < targetX && uiSlide) { xP += (targetX - xP) / 8; warningX = xP - 70; warningSymbolX = (warningX + warningWidth) / 2 - 728 / 2; angerMonitor.x = xP; joyMonitor.x = xP; fearMonitor.x = xP; contemptMonitor.x = xP; sadMonitor.x = xP + xgap; disgustMonitor.x = xP + xgap; surpriseMonitor.x = xP + xgap; } } } }
/** * Update the View and modules */ private void update() { Stopwatch stopwatch = new Stopwatch(); while (true) { if (model.SenseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) // Dauert manchmal voll lange ... { debug_y = 0; // <magic> PXCMCapture.Sample sample = model.SenseManager.QueryFaceSample(); sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData); model.FaceData = model.Face.CreateOutput(); model.FaceData.Update(); model.FaceAktuell = model.FaceData.QueryFaceByIndex(0); if (model.FaceAktuell != null) { uiSlide = true; PXCMFaceData.PoseData pose = model.FaceAktuell.QueryPose(); if (pose != null) { pose.QueryPoseAngles(out model.currentPose); } model.Lp = model.FaceAktuell.QueryLandmarks(); if (model.NullFace == null) { if (model.Lp != null) { PXCMFaceData.LandmarkPoint[] aPoints; model.Lp.QueryPoints(out aPoints); model.NullFace = aPoints; } } } colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); Graphics bitmapGraphics = Graphics.FromImage(colorBitmap); if (resetModules) { model.Modules.ForEach(delegate(RSModule mod) { mod.Reset(); }); resetModules = false; } if (testMode) { if (outputEnabled) { bitmapGraphics.FillRectangle(model.DefaultBGBrush, new Rectangle(0, 0, model.Width, model.Height)); } if (!imageEnabled) { bitmapGraphics.FillRectangle(model.OpaqueBGBrush, new Rectangle(0, 0, model.Width, model.Height)); } bitmapGraphics.DrawString("pose: " + model.CurrentPoseDiff, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); Debug_Y += 25; } else { using (Graphics gr = Graphics.FromImage(colorBitmap)) { if (model.calibrationProgress == 100) { angerMonitor.targetValue = (int)model.Emotions[Model.Emotion.ANGER]; fearMonitor.targetValue = (int)model.Emotions[Model.Emotion.FEAR]; disgustMonitor.targetValue = (int)model.Emotions[Model.Emotion.DISGUST]; surpriseMonitor.targetValue = (int)model.Emotions[Model.Emotion.SURPRISE]; joyMonitor.targetValue = (int)model.Emotions[Model.Emotion.JOY]; sadMonitor.targetValue = (int)model.Emotions[Model.Emotion.SADNESS]; contemptMonitor.targetValue = (int)model.Emotions[Model.Emotion.CONTEMPT]; angerMonitor.Step(); fearMonitor.Step(); disgustMonitor.Step(); surpriseMonitor.Step(); joyMonitor.Step(); sadMonitor.Step(); contemptMonitor.Step(); } if (!testMode) { gr.DrawImage(windowBitmap, xP - 90, yP - 150); } FriggnAweseomeGraphix.DrawMEMontior(gr, angerMonitor); FriggnAweseomeGraphix.DrawMEMontior(gr, sadMonitor); FriggnAweseomeGraphix.DrawMEMontior(gr, fearMonitor); FriggnAweseomeGraphix.DrawMEMontior(gr, surpriseMonitor); FriggnAweseomeGraphix.DrawMEMontior(gr, contemptMonitor); FriggnAweseomeGraphix.DrawMEMontior(gr, disgustMonitor); FriggnAweseomeGraphix.DrawMEMontior(gr, joyMonitor); bitmapGraphics.DrawString("Subject #" + subject, FriggnAweseomeGraphix.majorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP, yP - 75); bitmapGraphics.DrawLine(linePen, xP + 10, yP - 20, xP + 800, yP - 20); bitmapGraphics.DrawString("Pose: " + (int)model.CurrentPoseDiff, FriggnAweseomeGraphix.minorFont, new SolidBrush(FriggnAweseomeGraphix.fontColor), xP + 550, yP - 55); FriggnAweseomeGraphix.MEMonitor calibMonitor = new FriggnAweseomeGraphix.MEMonitor("", "", 1150, 580 - calibRadius, calibRadius, 20); calibMonitor.showPercent = false; if (model.calibrationProgress != 100 && model.CurrentFace != null) { PXCMFaceData.LandmarkPoint mPoint = model.CurrentFace[29]; int rad = 2; if (mPoint.image.x > 1200) { calibMonitor.y = (int)mPoint.image.y - calibRadius; calibMonitor.x = (int)mPoint.image.x - calibRadius; } calibMonitor.showPercent = false; calibMonitor.currentValue = (int)model.calibrationProgress; int sMark = (int)(model.calibrationProgress * 0.69); int tMark = sMark + 1; PXCMFaceData.LandmarkPoint sPoint = model.CurrentFace[sMark]; PXCMFaceData.LandmarkPoint tPoint = model.CurrentFace[tMark]; SolidBrush sb = new SolidBrush(FriggnAweseomeGraphix.fgColor); for (int i = 0; i < sMark - 1; i++) { PXCMFaceData.LandmarkPoint iPoint = model.CurrentFace[i]; gr.FillEllipse(sb, new Rectangle((int)iPoint.image.x - rad, (int)iPoint.image.y - rad, rad * 2, rad * 2)); } gr.FillEllipse(sb, new Rectangle((int)sPoint.image.x - rad, (int)sPoint.image.y - rad, rad * 4, rad * 4)); FriggnAweseomeGraphix.DrawMEMontior(gr, calibMonitor); } else { calibMonitor.currentValue = 0; } if (model.CurrentPoseDiff > model.PoseMax && model.calibrationProgress == 100) { warningPopupX += (warningPopupSlideX - warningPopupX) / 3; gr.FillRectangle(new SolidBrush(Color.FromArgb(100, 255, 255, 255)), new Rectangle(warningX, warningY, warningWidth, warningHeight)); gr.DrawImage(smallWindowBitmap, warningPopupX, 900); } else { warningPopupX = 2500; } } } if (model.FaceData.QueryNumberOfDetectedFaces() > 0 && model.CurrentFace != null) { model.Modules.ForEach(delegate(RSModule mod) { mod.Work(bitmapGraphics); if (outputEnabled && mod.output != "") { bitmapGraphics.DrawString(mod.output, model.DefaultFont, model.DefaultStringBrush, 10, Debug_Y); Debug_Y += 25; // new row } }); } double pitchDiff = Math.Abs(model.currentPose.pitch - model.NullPose.pitch); double rollDiff = Math.Abs(model.currentPose.roll - model.NullPose.roll); double yawDiff = Math.Abs(model.currentPose.yaw - model.NullPose.yaw); model.CurrentPoseDiff = pitchDiff + rollDiff + yawDiff; model.CurrentRollDiff = rollDiff; model.CurrentPitchDiff = pitchDiff; model.CurrentYawDiff = yawDiff; // Update PictureBox if (testMode) { pb.Image = colorBitmap; } else { pb.Image = colorBitmap; // uiBitmap.Clone(new Rectangle(0, 0, uiBitmap.Width, uiBitmap.Height), uiBitmap.PixelFormat); } if (PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL != model.SenseManager.captureManager.device.QueryMirrorMode()) { model.SenseManager.captureManager.device.SetMirrorMode(PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL); //mirror } model.SenseManager.ReleaseFrame(); model.FaceData.Dispose(); // DONE! sample.color.ReleaseAccess(colorData); if (xP < targetX && uiSlide) { xP += (targetX - xP) / 8; warningX = xP - 70; warningSymbolX = (warningX + warningWidth) / 2 - 728 / 2; angerMonitor.x = xP; joyMonitor.x = xP; fearMonitor.x = xP; contemptMonitor.x = xP; sadMonitor.x = xP + xgap; disgustMonitor.x = xP + xgap; surpriseMonitor.x = xP + xgap; } } } }
/** * Updates the modules and their respective data and refreshes the UI to display the current frame (the ActionUnit- and Emotion-Data). */ public void udpateAndVisualizeData() { if (vlcControl.IsPlaying) { vlcControl.Pause(); } model.CurrentFace = faceRecording.getFace(currentFrame); model.NullFace = faceRecording.getNullFace(); RSModule.Init(model); foreach (RSModule rsm in microExpressionModules) { rsm.Work(null); } foreach (RSModule rsm in emotionModules) { rsm.Work(null); } monitors[0].currentValue = (int)model.Emotions[Model.Emotion.ANGER]; monitors[1].currentValue = (int)model.Emotions[Model.Emotion.JOY]; monitors[2].currentValue = (int)model.Emotions[Model.Emotion.FEAR]; monitors[3].currentValue = (int)model.Emotions[Model.Emotion.CONTEMPT]; monitors[4].currentValue = (int)model.Emotions[Model.Emotion.SADNESS]; monitors[5].currentValue = (int)model.Emotions[Model.Emotion.DISGUST]; monitors[6].currentValue = (int)model.Emotions[Model.Emotion.SURPRISE]; //Bitmap newImage = new Bitmap(Width, viewHeight); Graphics g = Graphics.FromImage(dataImage); g.Clear(System.Drawing.SystemColors.MenuBar); g.TextRenderingHint = TextRenderingHint.AntiAlias; foreach (FriggnAweseomeGraphix.MEMonitor monitor in monitors) { FriggnAweseomeGraphix.DrawMEMontior(g, monitor, false); } int yPos = (int)(gap * 1.5), yPos2 = yPos; int idx = 0; if (viewHeight != VIEW_TINY) { foreach (KeyValuePair <string, double> entry in model.AU_Values) { int xBase, yBase; if (idx++ > 7 && viewHeight != VIEW_LARGE) { yBase = yPos2; xBase = monitors[0].radius * 4 + gap + 420; yPos2 += gap; } else { yBase = yPos; yPos += gap; xBase = monitors[0].radius * 4 + gap + 100; } g.DrawString(entry.Key.Substring(entry.Key.IndexOf(".") + 1), textFont, textBrush, xBase, yBase - 5); g.DrawString((int)entry.Value + "", textFont, textBrush, xBase + 250, yBase - 5); } } g.DrawString(shortName, textFont, textBrush, monitors[0].radius * 8 + gap, 0); g.DrawString("Frame: " + currentFrame, textFont, textBrush, monitors[0].radius * 8 + gap + 250, 0); g.DrawLine(new Pen(textBrush), 0, Height - 1, Width, Height - 1); Invoke(pictureUpdate); }