public void Evaluate(int SpreadMax)
{
if (this.FInvalidateConnect)
{
if (runtime != null)
{
this.runtime.AllFrameReady -= KinectFaceNode_AllFrameReady;
}
if (this.FInRuntime.PluginIO.IsConnected)
{
//Cache runtime node
this.runtime = this.FInRuntime[0];
if (runtime != null)
{
this.FInRuntime[0].AllFrameReady += KinectFaceNode_AllFrameReady;
}
}
this.FInvalidateConnect = false;
}
if (this.FInvalidate)
{
this.FOutOK.SliceCount = this.trackedSkeletons.Count;
this.FOutPosition.SliceCount = this.trackedSkeletons.Count;
this.FOutRotation.SliceCount = this.trackedSkeletons.Count;
List <FaceTrackFrame> frames = new List <FaceTrackFrame>();
int cnt = 0;
foreach (int key in this.trackedSkeletons.Keys)
{
SkeletonFaceTracker sft = this.trackedSkeletons[key];
if (sft.frame != null)
{
frames.Add((FaceTrackFrame)sft.frame.Clone());
this.FOutOK[cnt] = sft.frame.TrackSuccessful;
this.FOutPosition[cnt] = new Vector3(sft.frame.Translation.X, sft.frame.Translation.Y, sft.frame.Translation.Z);
this.FOutRotation[cnt] = new Vector3(sft.frame.Rotation.X, sft.frame.Rotation.Y, sft.frame.Rotation.Z) * (float)VMath.DegToCyc;
EnumIndexableCollection <FeaturePoint, PointF> pp = sft.frame.GetProjected3DShape();
EnumIndexableCollection <FeaturePoint, Vector3DF> p = sft.frame.Get3DShape();
}
else
{
this.FOutOK[cnt] = false;
this.FOutPosition[cnt] = Vector3.Zero;
this.FOutRotation[cnt] = Vector3.Zero;
}
cnt++;
}
this.FOutFrame.AssignFrom(frames);
}
}
protected override void OnRender(DrawingContext drawingContext)
{
base.OnRender(drawingContext);
foreach (KeyValuePair <int, SkeletonFaceTracker> item in this.trackedSkeletons)
{
int i = item.Key;
SkeletonFaceTracker faceInformation = item.Value;
faceInformation.DrawFaceModel(drawingContext, i);
}
}
public void TrackFrame(ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, DepthImagePixel[] depthImage, IEnumerable <Skeleton> skeletons, int frameNumber)
{
if (_currentColorImageFormat != colorImageFormat)
{
_currentColorImageFormat = colorImageFormat;
//_colorMappedDepthPoints = CreateDepthImagePointsFromColorFormat(colorImageFormat);
}
//var mapper = this.Kinect.CoordinateMapper;
//mapper.MapColorFrameToDepthFrame(colorImageFormat, depthImageFormat, depthImage, _colorMappedDepthPoints);
foreach (var skeleton in skeletons)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.skeletonFaceTrackers.ContainsKey(skeleton.TrackingId))
{
var tracker = new SkeletonFaceTracker();
this.skeletonFaceTrackers.Add(skeleton.TrackingId, tracker);
FaceTrackers.Add(tracker);
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.skeletonFaceTrackers.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, null, skeleton);
skeletonFaceTracker.LastTrackedFrame = frameNumber;
}
}
}
this.RemoveOldTrackers(frameNumber);
RaiseFrameUpdated();
}
public void Evaluate(int SpreadMax)
{
if (this.FInvalidateConnect)
{
if (runtime != null)
{
this.runtime.AllFrameReady -= KinectFaceNode_AllFrameReady;
}
if (this.FInRuntime.PluginIO.IsConnected)
{
//Cache runtime node
this.runtime = this.FInRuntime[0];
if (runtime != null)
{
this.FInRuntime[0].AllFrameReady += KinectFaceNode_AllFrameReady;
}
}
this.FInvalidateConnect = false;
}
if (this.FInvalidate)
{
this.FOutOK.SliceCount = this.trackedSkeletons.Count;
this.FOutPosition.SliceCount = this.trackedSkeletons.Count;
this.FOutRotation.SliceCount = this.trackedSkeletons.Count;
this.FOutPts.SliceCount = this.trackedSkeletons.Count;
this.FOutPPTs.SliceCount = this.trackedSkeletons.Count;
int cnt = 0;
foreach (int key in this.trackedSkeletons.Keys)
{
SkeletonFaceTracker sft = this.trackedSkeletons[key];
if (sft.frame != null)
{
this.FOutOK[cnt] = sft.frame.TrackSuccessful;
this.FOutPosition[cnt] = new Vector3(sft.frame.Translation.X, sft.frame.Translation.Y, sft.frame.Translation.Z);
this.FOutRotation[cnt] = new Vector3(sft.frame.Rotation.X, sft.frame.Rotation.Y, sft.frame.Rotation.Z) * (float)VMath.DegToCyc;
EnumIndexableCollection <FeaturePoint, PointF> pp = sft.frame.GetProjected3DShape();
EnumIndexableCollection <FeaturePoint, Vector3DF> p = sft.frame.Get3DShape();
this.FOutPPTs[cnt].SliceCount = pp.Count;
this.FOutPts[cnt].SliceCount = p.Count;
this.FOutNormals[cnt].SliceCount = p.Count;
//Compute smoothed normals
Vector3[] norms = new Vector3[p.Count];
int[] inds = KinectRuntime.FACE_INDICES;
int tricount = inds.Length / 3;
for (int j = 0; j < tricount; j++)
{
int i1 = inds[j * 3];
int i2 = inds[j * 3 + 1];
int i3 = inds[j * 3 + 2];
Vector3 v1 = new Vector3(p[i1].X, p[i1].Y, p[i1].Z);
Vector3 v2 = new Vector3(p[i2].X, p[i2].Y, p[i2].Z);
Vector3 v3 = new Vector3(p[i3].X, p[i3].Y, p[i3].Z);
Vector3 faceEdgeA = v2 - v1;
Vector3 faceEdgeB = v1 - v3;
Vector3 norm = Vector3.Cross(faceEdgeB, faceEdgeA);
norms[i1] += norm;
norms[i2] += norm;
norms[i3] += norm;
}
for (int i = 0; i < pp.Count; i++)
{
this.FOutPPTs[cnt][i] = new Vector2(pp[i].X, pp[i].Y);
this.FOutPts[cnt][i] = new Vector3(p[i].X, p[i].Y, p[i].Z);
this.FOutNormals[cnt][i] = Vector3.Normalize(norms[i]);
}
FaceTriangle[] d = sft.frame.GetTriangles();
this.FOutIndices.SliceCount = d.Length * 3;
for (int i = 0; i < d.Length; i++)
{
this.FOutIndices[i * 3] = d[i].First;
this.FOutIndices[i * 3 + 1] = d[i].Second;
this.FOutIndices[i * 3 + 2] = d[i].Third;
}
}
else
{
this.FOutOK[cnt] = false;
this.FOutPosition[cnt] = new Vector3(sft.frame.Translation.X, sft.frame.Translation.Y, sft.frame.Translation.Z);
this.FOutRotation[cnt] = new Vector3(sft.frame.Rotation.X, sft.frame.Rotation.Y, sft.frame.Rotation.Z) * (float)VMath.DegToCyc;
this.FOutIndices.SliceCount = 0;
this.FOutPPTs[cnt].SliceCount = 0;
this.FOutPts[cnt].SliceCount = 0;
}
cnt++;
}
//this.FOutOK[0] = this.frm.TrackSuccessful;
//this.FOutPosition[0] = new Vector3(frm.Translation.X, frm.Translation.Y, frm.Translation.Z);
}
//this.FOutFrameIndex[0] = this.frameindex;
}
public static void SetupDataStore(Pipeline pipeline, string outputStorePath, string inputStorePath, bool showLive,
Microsoft.Psi.Kinect.v1.KinectSensor kinectSensor, SkeletonFaceTracker faceTracker, IProducer <IStreamingSpeechRecognitionResult> speechRecog)
{
string outputLogPath = null;
if (outputStorePath != null && outputStorePath != "")
{
if (!Directory.Exists(outputStorePath))
{
Directory.CreateDirectory(outputStorePath);
}
outputLogPath = outputStorePath;
}
Console.WriteLine(outputLogPath == null);
string inputLogPath = null;
if (inputStorePath != null && inputStorePath != "" && Directory.Exists(inputStorePath))
{
inputLogPath = inputStorePath;
}
Console.WriteLine(inputLogPath == null);
// Needed only for live visualization
DateTime startTime = DateTime.Now;
// Create a data store to log the data to if necessary. A data store is necessary
// only if output logging or live visualization are enabled.
Console.WriteLine(outputLogPath == null);
var dataStore = CreateDataStore(pipeline, outputLogPath, showLive);
Console.WriteLine(dataStore == null);
Console.WriteLine("dataStore is empty");
// For disk logging or live visualization only
if (dataStore != null)
{
// Log the microphone audio and recognition results
//kinectSensor.ColorImage.Write("Kiosk.KinectSensor.ColorImage", dataStore);
kinectSensor.Audio.Write("Kiosk.KinectSensor.Audio", dataStore);
//faceTracker.Write("Kiosk.FaceTracker", dataStore);
speechRecog.Write($"Kiosk.FinalRecognitionResults", dataStore);
Console.WriteLine("Stored the data here! ");
}
// Ignore this block if live visualization is not enabled
if (showLive)
{
// Create the visualization client
var visualizationClient = new VisualizationClient();
// Clear all data if the visualizer is already open
visualizationClient.ClearAll();
// Create the visualization client to visualize live data
visualizationClient.SetLiveMode(startTime);
// Plot the video stream in a new panel
//visualizationClient.AddXYPanel();
//kinectSensor.ColorImage.Show(visualizationClient);
// Plot the microphone audio stream in a new panel
//visualizationClient.AddTimelinePanel();
//kinectSensor.Audio.Show(visualizationClient);
// Plot the recognition results in a new panel
//visualizationClient.AddTimelinePanel();
//faceTracker.Show(visualizationClient);
// Plot the recognition results in a new panel
//visualizationClient.AddTimelinePanel();
//speechRecog.Show(visualizationClient);
}
}
public void Evaluate(int SpreadMax)
{
if (this.FInvalidateConnect)
{
if (runtime != null)
{
this.runtime.AllFrameReady -= KinectFaceNode_AllFrameReady;
}
if (this.FInRuntime.PluginIO.IsConnected)
{
//Cache runtime node
this.runtime = this.FInRuntime[0];
if (runtime != null)
{
this.FInRuntime[0].AllFrameReady += KinectFaceNode_AllFrameReady;
}
}
this.FInvalidateConnect = false;
}
if (this.FInvalidate)
{
this.FOutOK.SliceCount = this.trackedSkeletons.Count;
this.FOutPosition.SliceCount = this.trackedSkeletons.Count;
this.FOutRotation.SliceCount = this.trackedSkeletons.Count;
this.FOutPts.SliceCount = this.trackedSkeletons.Count;
this.FOutPPTs.SliceCount = this.trackedSkeletons.Count;
int cnt = 0;
foreach (int key in this.trackedSkeletons.Keys)
{
SkeletonFaceTracker sft = this.trackedSkeletons[key];
if (sft.frame != null)
{
this.FOutOK[cnt] = sft.frame.TrackSuccessful;
this.FOutPosition[cnt] = new Vector3(sft.frame.Translation.X, sft.frame.Translation.Y, sft.frame.Translation.Z);
this.FOutRotation[cnt] = new Vector3(sft.frame.Rotation.X, sft.frame.Rotation.Y, sft.frame.Rotation.Z) * INVTWOPI;
EnumIndexableCollection <FeaturePoint, PointF> pp = sft.frame.GetProjected3DShape();
EnumIndexableCollection <FeaturePoint, Vector3DF> p = sft.frame.Get3DShape();
this.FOutPPTs[cnt].SliceCount = pp.Count;
this.FOutPts[cnt].SliceCount = p.Count;
for (int i = 0; i < pp.Count; i++)
{
this.FOutPPTs[cnt][i] = new Vector2(pp[i].X, pp[i].Y);
this.FOutPts[cnt][i] = new Vector3(p[i].X, p[i].Y, p[i].Z);
}
FaceTriangle[] d = sft.frame.GetTriangles();
this.FOutIndices.SliceCount = d.Length * 3;
for (int i = 0; i < d.Length; i++)
{
this.FOutIndices[i * 3] = d[i].First;
this.FOutIndices[i * 3 + 1] = d[i].Second;
this.FOutIndices[i * 3 + 2] = d[i].Third;
}
}
else
{
this.FOutOK[cnt] = false;
this.FOutPosition[cnt] = new Vector3(sft.frame.Translation.X, sft.frame.Translation.Y, sft.frame.Translation.Z);
this.FOutRotation[cnt] = new Vector3(sft.frame.Rotation.X, sft.frame.Rotation.Y, sft.frame.Rotation.Z) * INVTWOPI;
this.FOutIndices.SliceCount = 0;
this.FOutPPTs[cnt].SliceCount = 0;
this.FOutPts[cnt].SliceCount = 0;
}
cnt++;
}
//this.FOutOK[0] = this.frm.TrackSuccessful;
//this.FOutPosition[0] = new Vector3(frm.Translation.X, frm.Translation.Y, frm.Translation.Z);
}
//this.FOutFrameIndex[0] = this.frameindex;
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
if (TheStore.faceTrack == true)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
//TheSys.showError(DateTime.Now.ToString("hh:mm:ss.fff tt"));
using (colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame())
{
using (depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame())
{
using (skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame())
{
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
current_faceData = skeletonFaceTracker;
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
this.InvalidateVisual();
}
}
}
//TheSys.showError(DateTime.Now.ToString("====================="));
}
catch (Exception ex) { TheSys.showError(ex); }
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}
else
{
this.ResetFaceTracking();
}
}
private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)
{
ColorImageFrame colorImageFrame = null;
DepthImageFrame depthImageFrame = null;
SkeletonFrame skeletonFrame = null;
try
{
colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame();
depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame();
skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame();
if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)
{
return;
}
// Check for image format changes. The FaceTracker doesn't
// deal with that so we need to reset.
if (this.depthImageFormat != depthImageFrame.Format)
{
this.ResetFaceTracking();
this.depthImage = null;
this.depthImageFormat = depthImageFrame.Format;
}
if (this.colorImageFormat != colorImageFrame.Format)
{
this.ResetFaceTracking();
this.colorImage = null;
this.colorImageFormat = colorImageFrame.Format;
}
// Create any buffers to store copies of the data we work with
if (this.depthImage == null)
{
this.depthImage = new short[depthImageFrame.PixelDataLength];
}
if (this.colorImage == null)
{
this.colorImage = new byte[colorImageFrame.PixelDataLength];
}
// Get the skeleton information
if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)
{
this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];
}
colorImageFrame.CopyPixelDataTo(this.colorImage);
depthImageFrame.CopyPixelDataTo(this.depthImage);
skeletonFrame.CopySkeletonDataTo(this.skeletonData);
// Update the list of trackers and the trackers with the current frame information
foreach (Skeleton skeleton in this.skeletonData)
{
if (skeleton.TrackingState == SkeletonTrackingState.Tracked ||
skeleton.TrackingState == SkeletonTrackingState.PositionOnly)
{
// We want keep a record of any skeleton, tracked or untracked.
if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))
{
SkeletonFaceTracker tracker = new SkeletonFaceTracker(imageContainer, Kinect);
tracker.AccessoryImageSet = ChooseAccessory();
tracker.Skeleton = skeleton;
this.trackedSkeletons.Add(skeleton.TrackingId, tracker);
}
// Give each tracker the upated frame.
SkeletonFaceTracker skeletonFaceTracker;
if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))
{
skeletonFaceTracker.OnFrameReady(this.Kinect, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);
skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;
}
}
}
this.RemoveOldTrackers(skeletonFrame.FrameNumber);
if (trackedSkeletons.Count > 0)
{
if (this.SkeletonsTracked != null)
{
this.SkeletonsTracked.Invoke(this, new FaceInFocusEventHandler(this.skeletonData));
}
}
else
{
if (this.SkeletonsLost != null)
{
this.SkeletonsLost(this, new EventArgs());
}
}
this.InvalidateVisual();
}
finally
{
if (colorImageFrame != null)
{
colorImageFrame.Dispose();
}
if (depthImageFrame != null)
{
depthImageFrame.Dispose();
}
if (skeletonFrame != null)
{
skeletonFrame.Dispose();
}
}
}