// object owning data should be locked
private static void UpdateData(EventTypePluginData data)
{
Debug.Assert(data != null);
if ((data.sharedBodyIndexFrame != null))
{
if (data.rawBodyIndexTexture != null)
{
unsafe
{
data.rawBodyIndexTexture.UpdateData((byte *)data.sharedBodyIndexFrame.Buffer, data.sharedBodyIndexFrame.Size);
}
}
}
if (data.rawBodyIndexTexture != null)
{
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.RampConversion(data.rawBodyIndexTexture, data.rampTexture3d);
}
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.RampConversion(data.rawBodyIndexTexture, data.rampTexture2d);
}
}
}
// object owning data should be locked
private static void CreateTextures(EventTypePluginData data, viz.Context context)
{
Debug.Assert(data != null);
if (data.depthTexture != null)
{
data.depthTexture.Dispose();
data.depthTexture = null;
}
if (data.depthMap != null)
{
data.depthMap.Dispose();
data.depthMap = null;
}
if (data.uvTable != null)
{
data.uvTable.Dispose();
data.uvTable = null;
}
data.uvTable = new HGlobalBuffer(data.imageWidth * data.imageHeight * 2 * sizeof(float));
if (context != null)
{
data.depthTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R8G8B8A8_UNORM, true); // need to be created as render target for GPU conversion
data.depthMap = new viz.DepthMap(context, data.imageWidth, data.imageHeight, DepthPlugin.cFovX, DepthPlugin.cFovY, DepthPlugin.cMinZmm, DepthPlugin.cMaxZmm);
}
}
// object owning data should be locked
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (doAll)
{
if (data.timer != null)
{
data.timer.Stop();
}
// don't dispose this because others may be sharing the data
data.sharedColorFrame = null;
data.rgbaFrame = null;
data.decompressMemoryStreamBuffer = null;
data.convertColorBuffer = null;
}
if (data.colorTexture != null)
{
data.colorTexture.Dispose();
data.colorTexture = null;
}
}
// object owning data should be locked
private static bool UpdateSelectedPixelValue(EventTypePluginData data)
{
Debug.Assert(data != null);
bool changed = false;
if ((data.selected2DPixelX < data.imageWidth) && (data.selected2DPixelY < data.imageHeight))
{
uint offset = ((data.imageWidth * data.selected2DPixelY) + data.selected2DPixelX);
if (data.irFrame != null)
{
ushort temp = data.irFrame[offset];
if (temp != data.selected2DPixelIntensity)
{
changed = true;
data.selected2DPixelIntensity = temp;
}
}
if (data.depthFrame != null)
{
ushort temp = data.depthFrame[offset];
if (temp != data.selected2DPixelDepth)
{
changed = true;
data.selected2DPixelDepth = temp;
}
}
}
return(changed);
}
// object owning data should be locked
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (doAll)
{
// don't dispose this because others may be sharing the data
data.sharedDepthFrame = null;
}
if (data.depthTexture != null)
{
data.depthTexture.Dispose();
data.depthTexture = null;
}
if (data.depthMap != null)
{
data.depthMap.Dispose();
data.depthMap = null;
}
if (data.uvTable != null)
{
data.uvTable.Dispose();
data.uvTable = null;
}
}
private void ClearEvents(EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
doEvent = this.selectedData == data;
data.sharedIrFrame = null;
if (data.irTexture != null)
{
data.irTexture.Dispose();
data.irTexture = null;
data.imageWidth = 0;
data.imageHeight = 0;
}
data.selected2DPixelX = 0;
data.selected2DPixelY = 0;
data.selected2DPixelIrIntensity = 0;
}
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
this.RaisePropertyChanged("Selected2DPixelX");
this.RaisePropertyChanged("Selected2DPixelY");
this.RaisePropertyChanged("Selected2DPixelIrIntensity");
}
}
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (data.context != null)
{
data.context.Dispose();
data.context = null;
}
if (data.imageContext != null)
{
data.imageContext.Dispose();
data.imageContext = null;
}
if (doAll)
{
if (data.registration != null)
{
data.registration.Dispose();
data.registration = null;
}
}
}
// object owning data should be locked
private static bool UpdateSelectedPixelValue(EventTypePluginData data)
{
Debug.Assert(data != null);
bool changed = false;
if ((data.selected2DPixelX < data.imageWidth) && (data.selected2DPixelY < data.imageHeight))
{
uint offset = ((data.imageWidth * data.selected2DPixelY) + data.selected2DPixelX);
if (data.sharedIrFrame != null)
{
unsafe
{
ushort *p = (ushort *)data.sharedIrFrame.Buffer.ToPointer();
if (p != null)
{
ushort temp = p[offset];
if (temp != data.selected2DPixelIrIntensity)
{
changed = true;
data.selected2DPixelIrIntensity = temp;
}
}
}
}
}
return(changed);
}
// object owning data should be locked
private static void UpdateData(RawIrPlugin2DViewSettings pluginViewSettings, EventTypePluginData data)
{
Debug.Assert(data != null);
if ((pluginViewSettings != null) && (data.depthMap != null) && (data.depthTexture != null) && (data.irTexture != null))
{
switch (pluginViewSettings.ViewType)
{
case RawIrPlugin2DViewSettings.RawIr2DViewType.DepthColor:
data.depthMap.RampConversion(data.depthTexture, viz.DepthRampMode.Color);
break;
case RawIrPlugin2DViewSettings.RawIr2DViewType.DepthGrey:
data.depthMap.RampConversion(data.depthTexture, viz.DepthRampMode.Grey);
break;
case RawIrPlugin2DViewSettings.RawIr2DViewType.Ir:
data.irTexture.RampConversion(data.rawIrTexture, pluginViewSettings.RampTexture);
break;
default:
Debug.Assert(false);
break;
}
}
}
// object owning data should be locked
private static void CreateTextures(EventTypePluginData data, viz.Context context)
{
Debug.Assert(data != null);
if (data.rawBodyIndexTexture != null)
{
data.rawBodyIndexTexture.Dispose();
data.rawBodyIndexTexture = null;
}
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.Dispose();
data.bodyIndexTexture3d = null;
}
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.Dispose();
data.bodyIndexTexture2d = null;
}
if ((context != null) && (data.imageWidth > 0) && (data.imageHeight > 0))
{
data.bodyIndexTexture3d = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.B8G8R8A8_UNORM, true);
data.bodyIndexTexture2d = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.B8G8R8A8_UNORM, true);
data.rawBodyIndexTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R8_UNORM, false);
}
}
private void HandleEvent(EventType eventType, KStudioEvent eventObj, EventTypePluginData data)
{
Debug.Assert(data != null);
if (eventObj != null)
{
if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.Body) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.BodyMonitor))
{
lock (this.lockObj)
{
data.sharedBodyFrame = eventObj.GetRetainableEventDataBuffer();
uint bufferSize;
IntPtr bufferPtr;
eventObj.AccessUnderlyingEventDataBuffer(out bufferSize, out bufferPtr);
if (Marshal.SizeOf(typeof(nui.BODY_FRAME)) == bufferSize)
{
data.body.UpdateData(bufferPtr);
data.bodiesValid = true;
}
}
}
}
}
private void InitializeRender(viz.Context context, EventTypePluginData data)
{
Debug.Assert(data != null);
lock (this.lockObj)
{
Debug.Assert(data.rampTexture3d == null);
Debug.Assert(data.rampTexture2d == null);
if (context != null)
{
BodyIndexPlugin.CreateTextures(data, context);
// 2D ramp texture is different from 3D ramp texture for visualization with skeleton
uint[] rampData = new uint[BodyIndexPlugin.cRampTextureLength];
rampData[0] = 0xFFFF0000;
rampData[1] = 0xFF00FF00;
rampData[2] = 0xFF40FFFF;
rampData[3] = 0xFFFFFF40;
rampData[4] = 0xFFFF40FF;
rampData[5] = 0xFF8080FF;
for (int i = 6; i < rampData.Length; ++i)
{
rampData[i] = 0xFFFFFFFF;
}
data.rampTexture3d = new viz.Texture(context, (uint)rampData.Length, 1, viz.TextureFormat.B8G8R8A8_UNORM, false);
unsafe
{
fixed(uint *pRampData = rampData)
{
data.rampTexture3d.UpdateData((byte *)pRampData, (uint)rampData.Length * sizeof(uint));
}
}
rampData[0] = 0xC0FF0000;
rampData[1] = 0xC000FF00;
rampData[2] = 0xC040FFFF;
rampData[3] = 0xC0FFFF40;
rampData[4] = 0xC0FF40FF;
rampData[5] = 0xC08080FF;
for (int i = 6; i < rampData.Length; i++)
{
rampData[i] = 0xFF000000;
}
data.rampTexture2d = new viz.Texture(context, (uint)rampData.Length, 1, viz.TextureFormat.B8G8R8A8_UNORM, false);
unsafe
{
fixed(uint *pRampData = rampData)
{
data.rampTexture2d.UpdateData((byte *)pRampData, (uint)rampData.Length * sizeof(uint));
}
}
}
}
}
private void HandleEvent(KStudioEvent eventObj, EventTypePluginData data, EventType eventType)
{
if ((eventObj != null) && (data != null))
{
if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.BodyIndex) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.BodyIndexMonitor))
{
bool doDataEvent = false;
bool doVisibleEvent = false;
lock (this.lockObj)
{
bool isSelectedData = (this.selectedData == data);
HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer();
doVisibleEvent = isSelectedData && (newSharedFrame == null) != (data.sharedBodyIndexFrame == null);
data.sharedBodyIndexFrame = newSharedFrame;
uint newWidth = nui.Constants.STREAM_BODY_INDEX_WIDTH;
uint newHeight = nui.Constants.STREAM_BODY_INDEX_HEIGHT;
if ((data.rawBodyIndexTexture == null) || (newWidth != data.imageWidth) || (newHeight != data.imageHeight))
{
data.imageWidth = newWidth;
data.imageHeight = newHeight;
viz.Context context = null;
if (this.pluginService != null)
{
context = this.pluginService.GetContext(eventType);
}
if (context != null)
{
BodyIndexPlugin.CreateTextures(data, context);
}
}
if (BodyIndexPlugin.UpdateSelectedPixelValue(data))
{
doDataEvent = isSelectedData;
}
}
if (doVisibleEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
if (doDataEvent)
{
this.RaisePropertyChanged("Selected2DPixelBodyIndex");
}
}
}
}
private viz.Texture GetTexture(EventTypePluginData data)
{
Debug.Assert(data != null);
viz.Texture value = null;
value = this.UpdateData(data);
return(value);
}
private void InitializeRender(viz.Context context, EventTypePluginData data)
{
Debug.Assert(data != null);
lock (this.lockObj)
{
if (data.sharedColorFrame != null)
{
data.needsUpdate = true;
}
}
}
private void HandleEvent(KStudioEvent eventObj, EventTypePluginData data, EventType eventType)
{
Debug.Assert(data != null);
if (eventObj != null)
{
if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.UncompressedColor) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.UncompressedColorMonitor) ||
(eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.CompressedColor) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.CompressedColorMonitor))
{
lock (this.lockObj)
{
data.needsUpdate = true;
HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer();
data.visibleChanged = (newSharedFrame == null) != (data.sharedColorFrame == null);
data.sharedColorFrame = newSharedFrame;
data.doDecompress =
(eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.CompressedColor) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.CompressedColorMonitor);
uint newWidth = nui.Constants.STREAM_COLOR_WIDTH;
uint newHeight = nui.Constants.STREAM_COLOR_HEIGHT;
if ((newWidth != data.imageWidth) || (newHeight != data.imageHeight))
{
data.imageWidth = newWidth;
data.imageHeight = newHeight;
viz.Context context = null;
if (this.pluginService != null)
{
context = this.pluginService.GetContext(eventType);
}
ColorPlugin.CreateTextures(data, context);
}
if (!data.timer.IsEnabled)
{
data.timer.Start();
}
}
}
}
}
private void UpdatePropertyView(double x, double y, uint width, uint height, EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
if (this.selectedData != data)
{
doEvent = true;
this.selectedData = data;
}
data.selected2DPixelX = 0;
data.selected2DPixelY = 0;
data.selected2DPixelIntensity = 0;
data.selected2DPixelDepth = 0;
if (width == data.imageWidth)
{
data.selected2DPixelX = (uint)x;
}
else
{
data.selected2DPixelX = (uint)((x / width) * data.imageWidth);
}
if (height == data.imageHeight)
{
data.selected2DPixelY = (uint)y;
}
else
{
data.selected2DPixelY = (uint)((y / height) * data.imageHeight);
}
RawIrPlugin.UpdateSelectedPixelValue(data);
}
this.RaisePropertyChanged("Selected2DPixelX");
this.RaisePropertyChanged("Selected2DPixelY");
this.RaisePropertyChanged("Selected2DPixelIrIntensity");
this.RaisePropertyChanged("Selected2DPixelDepth");
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
}
private void UninitializeNuiViz(EventType eventType, EventTypePluginData data)
{
DebugHelper.AssertUIThread();
Debug.Assert(data != null);
PluginService.DisposeData(data, false);
if (this.plugins != null)
{
foreach (IImageVisualPlugin imagePlugin in this.plugins.OfType <IImageVisualPlugin>())
{
imagePlugin.UninitializeRender(eventType);
}
}
}
// object owning data should be locked
private static void CreateTextures(EventTypePluginData data, viz.Context context)
{
Debug.Assert(data != null);
if (data.colorTexture != null)
{
data.colorTexture.Dispose();
data.colorTexture = null;
}
data.rgbaFrame = new byte[data.imageWidth * data.imageHeight * sizeof(uint)];
if (context != null)
{
data.colorTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R8G8B8A8_UNORM, false);
}
}
private void InitializeNuiViz(EventType eventType, EventTypePluginData data)
{
DebugHelper.AssertUIThread();
Debug.Assert(data != null);
if ((this.plugins != null) && (data.context == null))
{
bool okay = false;
viz.Context context = new viz.Context();
if (context.HasInitialized())
{
data.imageContext = new viz.D3DImageContext();
data.context = context;
foreach (IImageVisualPlugin imagePlugin in this.plugins.OfType <IImageVisualPlugin>())
{
imagePlugin.InitializeRender(eventType, context);
}
okay = true;
}
else
{
context.Dispose();
}
if (!okay)
{
if (this.loggingService != null)
{
this.loggingService.LogLine(Strings.Error_LimitedGraphics);
}
if (this.notificationService != null)
{
Dispatcher.CurrentDispatcher.BeginInvoke(new Action(() =>
{
this.notificationService.ShowMessageBox(Strings.Error_LimitedGraphics);
}));
}
}
}
}
// object owning data should be locked
private static void Render3D(IPluginViewSettings pluginViewSettings, EventTypePluginData data, nui.Registration registration)
{
Debug.Assert(data != null);
BodyPlugin3DViewSettings bodyPluginViewSettings = pluginViewSettings as BodyPlugin3DViewSettings;
if ((bodyPluginViewSettings != null) && (registration != null) && (data.body != null) && data.bodiesValid)
{
if (bodyPluginViewSettings.RenderBodies || bodyPluginViewSettings.RenderHands)
{
data.body.Begin();
for (uint i = 0; i < BodyPlugin.bodyOptions.Length; ++i)
{
if (bodyPluginViewSettings.RenderBodies)
{
BodyOptions bodyOption = BodyPlugin.bodyOptions[i];
data.body.RenderBones(i, bodyOption.BoneEffect);
if (bodyPluginViewSettings.RenderJointOrientations)
{
data.body.RenderJointOrientations(i, bodyOption.JointEffect);
}
data.body.RenderJoints(i, bodyOption.JointEffect);
if (bodyPluginViewSettings.RenderInfo && (data.font != null))
{
// PlayderIndex (sic)
data.body.RenderInfo(i, viz.BodyInfoFlag.PlayderIndex, data.font, bodyOption.ColorVector);
}
}
if (bodyPluginViewSettings.RenderHands)
{
data.body.RenderHandStates(i);
}
}
data.body.End(0.0f);
}
}
}
public void InitializeRender(EventType eventType, viz.Context context)
{
switch (eventType)
{
case EventType.Monitor:
Debug.Assert(this.monitorData != null);
break;
case EventType.Inspection:
Debug.Assert(this.inspectionData != null);
if (this.inspectionData.sharedRawFrame != null)
{
EventTypePluginData data = this.inspectionData;
RawIrPlugin.CreateTextures(this.inspectionData, context);
DepthIrEngine depthIrEngine = this.pluginService.DepthIrEngine;
if ((depthIrEngine != null) && (data.depthFrame != null) && (data.irFrame != null))
{
unsafe
{
fixed(ushort *pDepthFrame = &data.depthFrame[0], pIrFrame = &data.irFrame[0])
{
depthIrEngine.HandleEvent(data.sharedRawFrame.Buffer, data.sharedRawFrame.Size, pDepthFrame, pIrFrame, data.imageWidth, data.imageHeight);
uint imageDataSize = data.imageWidth * data.imageHeight * sizeof(ushort);
if (data.rawIrTexture != null)
{
data.rawIrTexture.UpdateData((byte *)pIrFrame, imageDataSize);
}
if (data.depthMap != null)
{
data.depthMap.UpdateData(pDepthFrame, imageDataSize);
}
}
}
}
}
break;
}
}
private viz.Texture GetTexture(IPluginViewSettings pluginViewSettings, EventTypePluginData data)
{
Debug.Assert(data != null);
viz.Texture value = null;
lock (this.lockObj)
{
IrPluginViewSettings irPluginViewSettings = pluginViewSettings as IrPluginViewSettings;
if (irPluginViewSettings != null)
{
IrPlugin.UpdateData(irPluginViewSettings.RampTexture, data);
value = data.irTexture;
}
}
return(value);
}
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture)
{
lock (this.lockObj)
{
EventTypePluginData data = null;
nui.Registration registration = null;
switch (eventType)
{
case EventType.Monitor:
data = this.monitorData;
break;
case EventType.Inspection:
data = this.inspectionData;
break;
}
if (data != null)
{
if (this.pluginService != null)
{
registration = this.pluginService.GetRegistration(eventType);
if (data.lastRegistration != registration)
{
data.lastRegistration = this.pluginService.GetRegistration(eventType);
if ((data.lastRegistration != null) && (data.depthMap != null))
{
uint xyTableSize;
IntPtr xyTable = data.lastRegistration.GetXYTable(out xyTableSize);
data.depthMap.UpdateXYTable(xyTable, xyTableSize);
}
}
}
DepthPlugin.Render3D(pluginViewSettings, texture, data, registration);
}
}
}
// object owning data should be locked
private static void UpdateData(viz.Texture rampTexture, EventTypePluginData data)
{
Debug.Assert(data != null);
if (rampTexture != null)
{
if ((data.sharedIrFrame != null) && (data.rawIrTexture != null))
{
unsafe
{
data.rawIrTexture.UpdateData((byte *)data.sharedIrFrame.Buffer, data.sharedIrFrame.Size);
}
}
if (data.irTexture != null)
{
data.irTexture.RampConversion(data.rawIrTexture, rampTexture);
}
}
}
private void HandleTimer(EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
data.timer.Stop();
if (data.hasValidData)
{
doEvent = true;
data.hasValidData = false;
}
}
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
}
// instance of data should be locked
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (doAll)
{
// don't dispose this because others may be sharing the data
data.sharedBodyIndexFrame = null;
}
if (data.rawBodyIndexTexture != null)
{
data.rawBodyIndexTexture.Dispose();
data.rawBodyIndexTexture = null;
}
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.Dispose();
data.bodyIndexTexture3d = null;
}
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.Dispose();
data.bodyIndexTexture2d = null;
}
if (data.rampTexture3d != null)
{
data.rampTexture3d.Dispose();
data.rampTexture3d = null;
}
if (data.rampTexture2d != null)
{
data.rampTexture2d.Dispose();
data.rampTexture2d = null;
}
}
private void ClearEvents(EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
doEvent = this.selectedData == data;
data.sharedBodyIndexFrame = null;
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.Dispose();
data.bodyIndexTexture2d = null;
}
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.Dispose();
data.bodyIndexTexture3d = null;
}
data.imageWidth = 0;
data.imageHeight = 0;
data.selected2DPixelX = 0;
data.selected2DPixelY = 0;
data.selected2DPixelBodyIndex = 0xff;
}
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
this.RaisePropertyChanged("Selected2DPixelX");
this.RaisePropertyChanged("Selected2DPixelY");
this.RaisePropertyChanged("Selected2DPixeBodyIndex");
}
}
private viz.Texture GetTexture(IPluginViewSettings pluginViewSettings, EventTypePluginData data)
{
Debug.Assert(data != null);
viz.Texture value = null;
lock (this.lockObj)
{
BodyIndexPlugin.UpdateData(data);
if (pluginViewSettings is BodyIndexPlugin2DViewSettings)
{
value = data.bodyIndexTexture2d;
}
else if (pluginViewSettings is BodyIndexPlugin3DViewSettings)
{
value = data.bodyIndexTexture3d;
}
}
return(value);
}
// object owning data should be locked
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (doAll)
{
// don't dispose this because others may be sharing the data
data.sharedIrFrame = null;
}
if (data.rawIrTexture != null)
{
data.rawIrTexture.Dispose();
data.rawIrTexture = null;
}
if (data.irTexture != null)
{
data.irTexture.Dispose();
data.irTexture = null;
}
}
// object owning data should be locked
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (doAll)
{
// don't dispose this because others may be sharing the data
data.sharedRawFrame = null;
}
if (data.rawIrTexture != null)
{
data.rawIrTexture.Dispose();
data.rawIrTexture = null;
}
if (data.irTexture != null)
{
data.irTexture.Dispose();
data.irTexture = null;
}
if (data.depthTexture != null)
{
data.depthTexture.Dispose();
data.depthTexture = null;
}
if (data.depthMap != null)
{
data.depthMap.Dispose();
data.depthMap = null;
}
if (data.uvTable != null)
{
data.uvTable.Dispose();
data.uvTable = null;
}
}
private void ClearEvents(EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
doEvent = this.selectedData == data;
data.sharedColorFrame = null;
if (data.colorTexture != null)
{
data.colorTexture.Dispose();
data.colorTexture = null;
data.imageWidth = 0;
data.imageHeight = 0;
}
data.selected2DPixelX = 0;
data.selected2DPixelY = 0;
data.selected2DPixelColor = Colors.Black;
}
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
this.RaisePropertyChanged("Selected2DPixelX");
this.RaisePropertyChanged("Selected2DPixelY");
this.RaisePropertyChanged("Selected2DPixelColor");
}
}
// object owning data should be locked
private static void CreateTextures(EventTypePluginData data, viz.Context context)
{
Debug.Assert(data != null);
if (data.rawBodyIndexTexture != null)
{
data.rawBodyIndexTexture.Dispose();
data.rawBodyIndexTexture = null;
}
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.Dispose();
data.bodyIndexTexture3d = null;
}
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.Dispose();
data.bodyIndexTexture2d = null;
}
if ((context != null) && (data.imageWidth > 0) && (data.imageHeight > 0))
{
data.bodyIndexTexture3d = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.B8G8R8A8_UNORM, true);
data.bodyIndexTexture2d = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.B8G8R8A8_UNORM, true);
data.rawBodyIndexTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R8_UNORM, false);
}
}
private viz.Texture GetTexture(IPluginViewSettings pluginViewSettings, EventTypePluginData data)
{
Debug.Assert(data != null);
viz.Texture value = null;
lock (this.lockObj)
{
RawIrPlugin2DViewSettings raw2DSettingsViewSettings = pluginViewSettings as RawIrPlugin2DViewSettings;
if (raw2DSettingsViewSettings != null)
{
RawIrPlugin.UpdateData(raw2DSettingsViewSettings, data);
if (raw2DSettingsViewSettings.ViewType == RawIrPlugin2DViewSettings.RawIr2DViewType.Ir)
{
value = data.irTexture;
}
else
{
value = data.depthTexture;
}
}
else
{
RawIrPlugin3DViewSettings raw3DSettingsViewSettings = pluginViewSettings as RawIrPlugin3DViewSettings;
if (raw3DSettingsViewSettings != null)
{
if (raw3DSettingsViewSettings.ViewType == RawIrPlugin3DViewSettings.RawIr3DViewType.Ir)
{
value = data.irTexture;
}
}
}
}
return value;
}
private void UpdatePropertyView(double x, double y, uint width, uint height, EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
if (this.selectedData != data)
{
doEvent = true;
this.selectedData = data;
}
data.selected2DPixelX = 0;
data.selected2DPixelY = 0;
data.selected2DPixelIntensity = 0;
data.selected2DPixelDepth = 0;
if (width == data.imageWidth)
{
data.selected2DPixelX = (uint)x;
}
else
{
data.selected2DPixelX = (uint)((x / width) * data.imageWidth);
}
if (height == data.imageHeight)
{
data.selected2DPixelY = (uint)y;
}
else
{
data.selected2DPixelY = (uint)((y / height) * data.imageHeight);
}
RawIrPlugin.UpdateSelectedPixelValue(data);
}
this.RaisePropertyChanged("Selected2DPixelX");
this.RaisePropertyChanged("Selected2DPixelY");
this.RaisePropertyChanged("Selected2DPixelIrIntensity");
this.RaisePropertyChanged("Selected2DPixelDepth");
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
}
// object owning data should be locked
private static bool UpdateSelectedPixelValue(EventTypePluginData data)
{
Debug.Assert(data != null);
bool changed = false;
if ((data.selected2DPixelX < data.imageWidth) && (data.selected2DPixelY < data.imageHeight))
{
uint offset = ((data.imageWidth * data.selected2DPixelY) + data.selected2DPixelX);
if (data.irFrame != null)
{
ushort temp = data.irFrame[offset];
if (temp != data.selected2DPixelIntensity)
{
changed = true;
data.selected2DPixelIntensity = temp;
}
}
if (data.depthFrame != null)
{
ushort temp = data.depthFrame[offset];
if (temp != data.selected2DPixelDepth)
{
changed = true;
data.selected2DPixelDepth = temp;
}
}
}
return changed;
}
// object owning data should be locked
private static void UpdateData(RawIrPlugin2DViewSettings pluginViewSettings, EventTypePluginData data)
{
Debug.Assert(data != null);
if ((pluginViewSettings != null) && (data.depthMap != null) && (data.depthTexture != null) && (data.irTexture != null))
{
switch (pluginViewSettings.ViewType)
{
case RawIrPlugin2DViewSettings.RawIr2DViewType.DepthColor:
data.depthMap.RampConversion(data.depthTexture, viz.DepthRampMode.Color);
break;
case RawIrPlugin2DViewSettings.RawIr2DViewType.DepthGrey:
data.depthMap.RampConversion(data.depthTexture, viz.DepthRampMode.Grey);
break;
case RawIrPlugin2DViewSettings.RawIr2DViewType.Ir:
data.irTexture.RampConversion(data.rawIrTexture, pluginViewSettings.RampTexture);
break;
default:
Debug.Assert(false);
break;
}
}
}
private void InitializeRender(viz.Context context, EventTypePluginData data)
{
Debug.Assert(data != null);
lock (this.lockObj)
{
Debug.Assert(data.rampTexture3d == null);
Debug.Assert(data.rampTexture2d == null);
if (context != null)
{
BodyIndexPlugin.CreateTextures(data, context);
// 2D ramp texture is different from 3D ramp texture for visualization with skeleton
uint[] rampData = new uint[BodyIndexPlugin.cRampTextureLength];
rampData[0] = 0xFFFF0000;
rampData[1] = 0xFF00FF00;
rampData[2] = 0xFF40FFFF;
rampData[3] = 0xFFFFFF40;
rampData[4] = 0xFFFF40FF;
rampData[5] = 0xFF8080FF;
for (int i = 6; i < rampData.Length; ++i)
{
rampData[i] = 0xFFFFFFFF;
}
data.rampTexture3d = new viz.Texture(context, (uint)rampData.Length, 1, viz.TextureFormat.B8G8R8A8_UNORM, false);
unsafe
{
fixed (uint* pRampData = rampData)
{
data.rampTexture3d.UpdateData((byte*)pRampData, (uint)rampData.Length * sizeof(uint));
}
}
rampData[0] = 0xC0FF0000;
rampData[1] = 0xC000FF00;
rampData[2] = 0xC040FFFF;
rampData[3] = 0xC0FFFF40;
rampData[4] = 0xC0FF40FF;
rampData[5] = 0xC08080FF;
for (int i = 6; i < rampData.Length; i++)
{
rampData[i] = 0xFF000000;
}
data.rampTexture2d = new viz.Texture(context, (uint)rampData.Length, 1, viz.TextureFormat.B8G8R8A8_UNORM, false);
unsafe
{
fixed (uint* pRampData = rampData)
{
data.rampTexture2d.UpdateData((byte*)pRampData, (uint)rampData.Length * sizeof(uint));
}
}
}
}
}
private viz.Texture GetTexture(IPluginViewSettings pluginViewSettings, EventTypePluginData data)
{
Debug.Assert(data != null);
viz.Texture value = null;
lock (this.lockObj)
{
BodyIndexPlugin.UpdateData(data);
if (pluginViewSettings is BodyIndexPlugin2DViewSettings)
{
value = data.bodyIndexTexture2d;
}
else if (pluginViewSettings is BodyIndexPlugin3DViewSettings)
{
value = data.bodyIndexTexture3d;
}
}
return value;
}
// object owning data should be locked
private static void CreateTextures(EventTypePluginData data, viz.Context context)
{
Debug.Assert(data != null);
if (data.colorTexture != null)
{
data.colorTexture.Dispose();
data.colorTexture = null;
}
data.rgbaFrame = new byte[data.imageWidth * data.imageHeight * sizeof(uint)];
if (context != null)
{
data.colorTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R8G8B8A8_UNORM, false);
}
}
// object owning data should be locked
private static void Render3D(IPluginViewSettings pluginViewSettings, EventTypePluginData data, nui.Registration registration)
{
Debug.Assert(data != null);
BodyPlugin3DViewSettings bodyPluginViewSettings = pluginViewSettings as BodyPlugin3DViewSettings;
if ((bodyPluginViewSettings != null) && (registration != null) && (data.body != null) && data.bodiesValid)
{
if (bodyPluginViewSettings.RenderBodies || bodyPluginViewSettings.RenderHands)
{
data.body.Begin();
for (uint i = 0; i < BodyPlugin.bodyOptions.Length; ++i)
{
if (bodyPluginViewSettings.RenderBodies)
{
BodyOptions bodyOption = BodyPlugin.bodyOptions[i];
data.body.RenderBones(i, bodyOption.BoneEffect);
if (bodyPluginViewSettings.RenderJointOrientations)
{
data.body.RenderJointOrientations(i, bodyOption.JointEffect);
}
data.body.RenderJoints(i, bodyOption.JointEffect);
if (bodyPluginViewSettings.RenderInfo && (data.font != null))
{
// PlayderIndex (sic)
data.body.RenderInfo(i, viz.BodyInfoFlag.PlayderIndex, data.font, bodyOption.ColorVector);
}
}
if (bodyPluginViewSettings.RenderHands)
{
data.body.RenderHandStates(i);
}
}
data.body.End(0.0f);
}
}
}
// object owning data should be locked
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (doAll)
{
if (data.timer != null)
{
data.timer.Stop();
}
// don't dispose this because others may be sharing the data
data.sharedColorFrame = null;
data.rgbaFrame = null;
data.decompressMemoryStreamBuffer = null;
data.convertColorBuffer = null;
}
if (data.colorTexture != null)
{
data.colorTexture.Dispose();
data.colorTexture = null;
}
}
private void InitializeRender(viz.Context context, EventTypePluginData data)
{
Debug.Assert(data != null);
lock (this.lockObj)
{
if (data.sharedColorFrame != null)
{
data.needsUpdate = true;
}
}
}
private void HandleEvent(KStudioEvent eventObj, EventTypePluginData data, EventType eventType)
{
Debug.Assert(data != null);
if (eventObj != null)
{
if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.UncompressedColor) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.UncompressedColorMonitor) ||
(eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.CompressedColor) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.CompressedColorMonitor))
{
lock (this.lockObj)
{
data.needsUpdate = true;
HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer();
data.visibleChanged = (newSharedFrame == null) != (data.sharedColorFrame == null);
data.sharedColorFrame = newSharedFrame;
data.doDecompress =
(eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.CompressedColor) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.CompressedColorMonitor);
uint newWidth = nui.Constants.STREAM_COLOR_WIDTH;
uint newHeight = nui.Constants.STREAM_COLOR_HEIGHT;
if ((newWidth != data.imageWidth) || (newHeight != data.imageHeight))
{
data.imageWidth = newWidth;
data.imageHeight = newHeight;
viz.Context context = null;
if (this.pluginService != null)
{
context = this.pluginService.GetContext(eventType);
}
ColorPlugin.CreateTextures(data, context);
}
if (!data.timer.IsEnabled)
{
data.timer.Start();
}
}
}
}
}
private void ClearEvents(EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
doEvent = this.selectedData == data;
data.sharedRawFrame = null;
data.imageWidth = 0;
data.imageHeight = 0;
data.depthFrame = null;
data.irFrame = null;
if (data.rawIrTexture != null)
{
data.rawIrTexture.Dispose();
data.rawIrTexture = null;
}
if (data.irTexture != null)
{
data.irTexture.Dispose();
data.irTexture = null;
}
if (data.depthTexture != null)
{
data.depthTexture.Dispose();
data.depthTexture = null;
}
if (data.depthMap != null)
{
data.depthMap.Dispose();
data.depthMap = null;
}
if (data.uvTable != null)
{
data.uvTable.Dispose();
data.uvTable = null;
}
data.uvTable = null;
}
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
this.RaisePropertyChanged("Selected2DPixelX");
this.RaisePropertyChanged("Selected2DPixelY");
this.RaisePropertyChanged("Selected2DPixelIrIntensity");
this.RaisePropertyChanged("Selected2DPixelDepth");
}
}
private void ClearEvents(EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
doEvent = this.selectedData == data;
data.sharedBodyIndexFrame = null;
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.Dispose();
data.bodyIndexTexture2d = null;
}
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.Dispose();
data.bodyIndexTexture3d = null;
}
data.imageWidth = 0;
data.imageHeight = 0;
data.selected2DPixelX = 0;
data.selected2DPixelY = 0;
data.selected2DPixelBodyIndex = 0xff;
}
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
this.RaisePropertyChanged("Selected2DPixelX");
this.RaisePropertyChanged("Selected2DPixelY");
this.RaisePropertyChanged("Selected2DPixeBodyIndex");
}
}
private void HandleEvent(EventType eventType, KStudioEvent eventObj, EventTypePluginData data)
{
Debug.Assert(data != null);
if (eventObj != null)
{
bool doDataEvent = false;
bool doVisibleEvent = false;
if (eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.RawIr)
{
lock (this.lockObj)
{
bool isSelectedData = (this.selectedData == data);
HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer();
doVisibleEvent = isSelectedData && (newSharedFrame == null) != (data.sharedRawFrame == null);
data.sharedRawFrame = newSharedFrame;
uint newWidth = nui.Constants.STREAM_IR_WIDTH;
uint newHeight = nui.Constants.STREAM_IR_HEIGHT;
if ((data.rawIrTexture == null) || (newWidth != data.imageWidth) || (newHeight != data.imageHeight))
{
data.imageWidth = newWidth;
data.imageHeight = newHeight;
viz.Context context = null;
if (this.pluginService != null)
{
context = this.pluginService.GetContext(eventType);
}
RawIrPlugin.CreateTextures(data, context);
}
DepthIrEngine depthIrEngine = this.pluginService.DepthIrEngine;
if ((depthIrEngine != null) && (data.depthFrame != null) && (data.irFrame != null))
{
unsafe
{
fixed (ushort* pDepthFrame = &data.depthFrame[0], pIrFrame = &data.irFrame[0])
{
depthIrEngine.HandleEvent(data.sharedRawFrame.Buffer, data.sharedRawFrame.Size, pDepthFrame, pIrFrame, data.imageWidth, data.imageHeight);
uint imageDataSize = data.imageWidth * data.imageHeight * sizeof(ushort);
if (data.rawIrTexture != null)
{
data.rawIrTexture.UpdateData((byte*)pIrFrame, imageDataSize);
}
if (data.depthMap != null)
{
data.depthMap.UpdateData(pDepthFrame, imageDataSize);
}
}
}
}
if (RawIrPlugin.UpdateSelectedPixelValue(data))
{
doDataEvent = isSelectedData;
}
}
if (doVisibleEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
if (doDataEvent)
{
this.RaisePropertyChanged("Selected2DPixelIrIntensity");
this.RaisePropertyChanged("Selected2DPixelDepth");
}
}
}
}
// object owning data should be locked
private static void Render2D(IPluginViewSettings pluginViewSettings, viz.Texture texture, float left, float top, float width, float height, EventTypePluginData data, nui.Registration registration)
{
Debug.Assert(data != null);
BodyPlugin2DViewSettings bodyPluginViewSettings = pluginViewSettings as BodyPlugin2DViewSettings;
if ((bodyPluginViewSettings != null) && (registration != null) && (data.body != null) && data.bodiesValid)
{
if (bodyPluginViewSettings.RenderBodies || bodyPluginViewSettings.RenderHands)
{
viz.Body2DMode bodyMode = viz.Body2DMode.DepthIR;
if ((texture != null) && (texture.GetWidth() == nui.Constants.STREAM_COLOR_WIDTH) && (texture.GetHeight() == nui.Constants.STREAM_COLOR_HEIGHT))
{
bodyMode = viz.Body2DMode.Color;
}
data.body.Begin2D(left, top, width, height, bodyMode, registration.GetCalibrationData());
for (uint i = 0; i < BodyPlugin.bodyOptions.Length; ++i)
{
if (bodyPluginViewSettings.RenderBodies)
{
viz.Vector color = BodyPlugin.bodyOptions[i].ColorVector;
color.A = 0.7f;
data.body.RenderBones2D(i, color);
}
if (bodyPluginViewSettings.RenderHands)
{
data.body.RenderHandStates2D(i);
}
}
data.body.End2D();
}
}
}
private static void UpdateData(RawIrPlugin3DViewSettings pluginViewSettings, viz.Texture texture, EventTypePluginData data)
{
Debug.Assert(data != null);
if ((pluginViewSettings != null) && (data.depthMap != null) && (data.depthTexture != null))
{
if (pluginViewSettings.IsSupplyingSurface)
{
data.depthMap.SetMode(viz.DepthVertexMode.Surface, viz.DepthRampMode.None);
}
else
{
switch (pluginViewSettings.ViewType)
{
case RawIrPlugin3DViewSettings.RawIr3DViewType.DepthColor:
data.depthMap.SetMode(viz.DepthVertexMode.Point, viz.DepthRampMode.Color);
break;
case RawIrPlugin3DViewSettings.RawIr3DViewType.DepthGrey:
data.depthMap.SetMode(viz.DepthVertexMode.Point, viz.DepthRampMode.Grey);
break;
case RawIrPlugin3DViewSettings.RawIr3DViewType.SurfaceNormal:
data.depthMap.SetMode(viz.DepthVertexMode.SurfaceWithNormal, viz.DepthRampMode.None);
break;
case RawIrPlugin3DViewSettings.RawIr3DViewType.Ir:
data.depthMap.SetMode(viz.DepthVertexMode.Surface, viz.DepthRampMode.None);
data.irTexture.RampConversion(data.rawIrTexture, pluginViewSettings.RampTexture);
break;
default:
Debug.Assert(false);
break;
}
}
}
}
// object owning data should be locked
private static bool UpdateSelectedPixelValue(EventTypePluginData data)
{
Debug.Assert(data != null);
bool changed = false;
if ((data.selected2DPixelX < data.imageWidth) && (data.selected2DPixelY < data.imageHeight))
{
uint offset = ((data.imageWidth * data.selected2DPixelY) + data.selected2DPixelX) * 4;
if (data.rgbaFrame != null)
{
changed = true;
byte red = data.rgbaFrame[offset];
byte green = data.rgbaFrame[offset + 1];
byte blue = data.rgbaFrame[offset + 2];
data.selected2DPixelColor = Color.FromRgb(red, green, blue);
}
}
return changed;
}
// object owning data should be locked
private static void CreateTextures(EventTypePluginData data, viz.Context context)
{
Debug.Assert(data != null);
if (data.rawIrTexture != null)
{
data.rawIrTexture.Dispose();
data.rawIrTexture = null;
}
if (data.irTexture != null)
{
data.irTexture.Dispose();
data.irTexture = null;
}
if (data.depthTexture != null)
{
data.depthTexture.Dispose();
data.depthTexture = null;
}
if (data.depthMap != null)
{
data.depthMap.Dispose();
data.depthMap = null;
}
if (data.uvTable != null)
{
data.uvTable.Dispose();
data.uvTable = null;
}
data.uvTable = new HGlobalBuffer(data.imageWidth * data.imageHeight * 2 * sizeof(float));
if (context != null)
{
data.rawIrTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R16_UNORM, false);
data.irTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R8G8B8A8_UNORM, true);
data.depthTexture = new viz.Texture(context, data.imageWidth, data.imageHeight, viz.TextureFormat.R8G8B8A8_UNORM, true); // need to be created as render target for GPU conversion
data.depthMap = new viz.DepthMap(context, data.imageWidth, data.imageHeight, RawIrPlugin.cFovX, RawIrPlugin.cFovY, RawIrPlugin.cMinZmm, RawIrPlugin.cMaxZmm);
}
data.depthFrame = new ushort[data.imageWidth * data.imageHeight];
data.irFrame = new ushort[data.imageWidth * data.imageHeight];
}
// object owning data should be locked
private static void UpdateData(EventTypePluginData data)
{
Debug.Assert(data != null);
if ((data.sharedBodyIndexFrame != null))
{
if (data.rawBodyIndexTexture != null)
{
unsafe
{
data.rawBodyIndexTexture.UpdateData((byte*)data.sharedBodyIndexFrame.Buffer, data.sharedBodyIndexFrame.Size);
}
}
}
if (data.rawBodyIndexTexture != null)
{
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.RampConversion(data.rawBodyIndexTexture, data.rampTexture3d);
}
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.RampConversion(data.rawBodyIndexTexture, data.rampTexture2d);
}
}
}
private static void Render3D(IPluginViewSettings pluginViewSettings, viz.Texture texture, EventTypePluginData data, nui.Registration registration)
{
Debug.Assert(data != null);
RawIrPlugin3DViewSettings rawPlugin3DViewSettings = pluginViewSettings as RawIrPlugin3DViewSettings;
if (rawPlugin3DViewSettings != null)
{
RawIrPlugin.UpdateData(rawPlugin3DViewSettings, texture, data);
bool doColor = false;
bool doBodyIndex = false;
if (texture != null)
{
uint textureWidth = texture.GetWidth();
uint textureHeight = texture.GetHeight();
doColor = (textureWidth == nui.Constants.STREAM_COLOR_WIDTH) &&
(textureHeight == nui.Constants.STREAM_COLOR_HEIGHT);
doBodyIndex = (texture.GetTextureFormat() == viz.TextureFormat.B8G8R8A8_UNORM) &&
(textureWidth == nui.Constants.STREAM_IR_WIDTH) &&
(textureHeight == nui.Constants.STREAM_IR_HEIGHT);
}
if ((!rawPlugin3DViewSettings.IsSupplyingSurface && (rawPlugin3DViewSettings.ViewType == RawIrPlugin3DViewSettings.RawIr3DViewType.Ir)) ||
(rawPlugin3DViewSettings.IsSupplyingSurface && doBodyIndex) ||
(!rawPlugin3DViewSettings.IsSupplyingSurface && rawPlugin3DViewSettings.ViewType == RawIrPlugin3DViewSettings.RawIr3DViewType.SurfaceNormal))
{
if (data.depthMap != null)
{
// special case for body index
viz.Effect effect = new viz.Effect()
{
Direction = new viz.Vector(0.5f, 0.3f, 1.5f, 0),
Ambient = new viz.Vector(0.0f, 0.0f, 0.0f, 1.0f),
Diffuse = new viz.Vector(0.5f, 0.5f, 0.5f, 1.0f),
Specular = new viz.Vector(1.0f, 1.0f, 1.0f, 1.0f),
Power = 25.0f,
EnableLighting = true,
EnableTexture = false,
};
if ((rawPlugin3DViewSettings.IsSupplyingSurface && doBodyIndex))
{
if (data.depthMap != null)
{
data.depthMap.SetMode(viz.DepthVertexMode.SurfaceWithNormal, viz.DepthRampMode.None);
effect.EnableTexture = true;
}
}
if (!rawPlugin3DViewSettings.IsSupplyingSurface && (rawPlugin3DViewSettings.ViewType != RawIrPlugin3DViewSettings.RawIr3DViewType.Ir))
{
texture = null;
}
data.depthMap.Render(effect, texture);
}
}
else
{
if (rawPlugin3DViewSettings.IsSupplyingSurface && doColor)
{
// special case for color
if ((registration != null) && (data.depthMap != null) && (data.uvTable != null) && (data.sharedRawFrame != null))
{
data.depthMap.SetMode(viz.DepthVertexMode.SurfaceWithUV, viz.DepthRampMode.None);
IntPtr ptr = data.sharedRawFrame.Buffer;
if (ptr != IntPtr.Zero)
{
registration.Process(ptr, data.uvTable.Buffer);
}
data.depthMap.UpdateUVTable(data.uvTable.Buffer, data.uvTable.Size);
}
}
if (data.depthMap != null)
{
viz.Effect effect = new viz.Effect()
{
EnableTexture = texture != null,
};
data.depthMap.Render(effect, texture);
}
}
}
}
// instance of data should be locked
private static void DisposeData(EventTypePluginData data, bool doAll)
{
Debug.Assert(data != null);
if (doAll)
{
// don't dispose this because others may be sharing the data
data.sharedBodyIndexFrame = null;
}
if (data.rawBodyIndexTexture != null)
{
data.rawBodyIndexTexture.Dispose();
data.rawBodyIndexTexture = null;
}
if (data.bodyIndexTexture3d != null)
{
data.bodyIndexTexture3d.Dispose();
data.bodyIndexTexture3d = null;
}
if (data.bodyIndexTexture2d != null)
{
data.bodyIndexTexture2d.Dispose();
data.bodyIndexTexture2d = null;
}
if (data.rampTexture3d != null)
{
data.rampTexture3d.Dispose();
data.rampTexture3d = null;
}
if (data.rampTexture2d != null)
{
data.rampTexture2d.Dispose();
data.rampTexture2d = null;
}
}
private void HandleEvent(EventType eventType, KStudioEvent eventObj, EventTypePluginData data)
{
Debug.Assert(data != null);
if (eventObj != null)
{
if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.Body) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.BodyMonitor))
{
lock (this.lockObj)
{
data.sharedBodyFrame = eventObj.GetRetainableEventDataBuffer();
uint bufferSize;
IntPtr bufferPtr;
eventObj.AccessUnderlyingEventDataBuffer(out bufferSize, out bufferPtr);
if (Marshal.SizeOf(typeof(nui.BODY_FRAME)) == bufferSize)
{
data.body.UpdateData(bufferPtr);
data.bodiesValid = true;
}
}
}
}
}
private void HandleTimer(EventTypePluginData data)
{
Debug.Assert(data != null);
bool doEvent = false;
lock (this.lockObj)
{
data.timer.Stop();
if (data.hasValidData)
{
doEvent = true;
data.hasValidData = false;
}
}
if (doEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
}
private void HandleEvent(KStudioEvent eventObj, EventTypePluginData data, EventType eventType)
{
if ((eventObj != null) && (data != null))
{
if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.BodyIndex) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.BodyIndexMonitor))
{
bool doDataEvent = false;
bool doVisibleEvent = false;
lock (this.lockObj)
{
bool isSelectedData = (this.selectedData == data);
HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer();
doVisibleEvent = isSelectedData && (newSharedFrame == null) != (data.sharedBodyIndexFrame == null);
data.sharedBodyIndexFrame = newSharedFrame;
uint newWidth = nui.Constants.STREAM_BODY_INDEX_WIDTH;
uint newHeight = nui.Constants.STREAM_BODY_INDEX_HEIGHT;
if ((data.rawBodyIndexTexture == null) || (newWidth != data.imageWidth) || (newHeight != data.imageHeight))
{
data.imageWidth = newWidth;
data.imageHeight = newHeight;
viz.Context context = null;
if (this.pluginService != null)
{
context = this.pluginService.GetContext(eventType);
}
if (context != null)
{
BodyIndexPlugin.CreateTextures(data, context);
}
}
if (BodyIndexPlugin.UpdateSelectedPixelValue(data))
{
doDataEvent = isSelectedData;
}
}
if (doVisibleEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
if (doDataEvent)
{
this.RaisePropertyChanged("Selected2DPixelBodyIndex");
}
}
}
}
private viz.Texture UpdateData(EventTypePluginData data)
{
Debug.Assert(data != null);
viz.Texture value = null;
bool doDataEvent = false;
bool doVisibleEvent = false;
lock (this.lockObj)
{
bool updated = false;
value = data.colorTexture;
bool isSelectedData = (this.selectedData == data);
data.visibleChanged = false;
if (data.needsUpdate && (data.sharedColorFrame != null) && (data.sharedColorFrame.Buffer != IntPtr.Zero) && (data.colorTexture != null))
{
try
{
if (data.doDecompress)
{
DateTime now = DateTime.Now;
if (now > this.nextDecompress)
{
data.timer.Stop();
doVisibleEvent = isSelectedData && (!data.hasValidData || data.visibleChanged);
data.needsUpdate = false;
data.hasValidData = true;
updated = true;
if ((data.decompressMemoryStreamBuffer == null) || (data.decompressMemoryStreamBuffer.Length < data.sharedColorFrame.Size))
{
data.decompressMemoryStreamBuffer = new byte[data.sharedColorFrame.Size];
}
uint convertBufferSize = data.imageWidth * data.imageHeight * 3;
if ((data.convertColorBuffer == null) || (data.convertColorBuffer.Length < convertBufferSize))
{
data.convertColorBuffer = new byte[convertBufferSize];
}
byte[] rgbBuffer = data.convertColorBuffer;
if (this.averageDecompressCount > 10)
{
this.averageDecompressCount = 1;
this.stopwatch.Reset();
}
else
{
this.averageDecompressCount++;
}
this.stopwatch.Start();
Marshal.Copy(data.sharedColorFrame.Buffer, data.decompressMemoryStreamBuffer, 0, (int)data.sharedColorFrame.Size);
using (Stream imageStream = new MemoryStream(data.decompressMemoryStreamBuffer))
{
// this has to run on the UI thread
JpegBitmapDecoder decoder = new JpegBitmapDecoder(imageStream, BitmapCreateOptions.PreservePixelFormat, BitmapCacheOption.Default);
BitmapSource bitmapSource = decoder.Frames[0];
bitmapSource.CopyPixels(rgbBuffer, (int)(data.imageWidth * 3), 0);
// expand bgr to rgba
uint imageSize = data.imageWidth * data.imageHeight;
for (uint index = 0; index < imageSize; ++index)
{
uint rgbaIndex = index * 4;
uint bgrIndex = index * 3;
data.rgbaFrame[rgbaIndex] = rgbBuffer[bgrIndex + 2];
data.rgbaFrame[rgbaIndex + 1] = rgbBuffer[bgrIndex + 1];
data.rgbaFrame[rgbaIndex + 2] = rgbBuffer[bgrIndex];
data.rgbaFrame[rgbaIndex + 3] = 0xFF;
}
}
this.stopwatch.Stop();
// color decompression takes a LONG time (sometimes upwards of 1/10 of second), so throttle it
double averageDecompressTime = ((double)this.stopwatch.ElapsedMilliseconds) / this.averageDecompressCount;
double sinceLastDecompress = (now - this.lastDecompress).TotalMilliseconds;
if (sinceLastDecompress > 1000)
{
this.nextDecompress = now + TimeSpan.FromMilliseconds(averageDecompressTime * 2);
}
else
{
double minTime = Math.Max(sinceLastDecompress, averageDecompressTime * 2);
this.nextDecompress = now + TimeSpan.FromMilliseconds(minTime);
}
this.lastDecompress = now;
}
}
else
{
data.timer.Stop();
doVisibleEvent = isSelectedData && (!data.hasValidData || data.visibleChanged);
data.needsUpdate = false;
data.hasValidData = true;
updated = true;
unsafe
{
fixed (byte* p = data.rgbaFrame)
{
GCHandle rgbaFrameHandle = GCHandle.Alloc(data.rgbaFrame, GCHandleType.Pinned);
NativeMethods.ConvertYUYVToRGBA(
data.sharedColorFrame.Buffer,
data.sharedColorFrame.Size,
rgbaFrameHandle.AddrOfPinnedObject(),
(uint)data.rgbaFrame.Length);
}
}
}
}
catch (Exception)
{
// TODO_LOG
}
}
if (updated)
{
if (ColorPlugin.UpdateSelectedPixelValue(data))
{
doDataEvent = isSelectedData;
}
if ((data.colorTexture != null) && (data.rgbaFrame != null))
{
unsafe
{
fixed (byte* pFrame = data.rgbaFrame)
{
data.colorTexture.UpdateData(pFrame, (uint)data.rgbaFrame.Length);
}
}
}
}
}
if (doVisibleEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
if (doDataEvent)
{
this.RaisePropertyChanged("Selected2DPixelColor");
}
return value;
}
// this.IsPlaybackFileOnTarge data should be locked
private static bool UpdateSelectedPixelValue(EventTypePluginData data)
{
Debug.Assert(data != null);
bool changed = false;
if ((data.selected2DPixelX < data.imageWidth) && (data.selected2DPixelY < data.imageHeight))
{
uint offset = ((data.imageWidth * data.selected2DPixelY) + data.selected2DPixelX);
if (data.sharedBodyIndexFrame != null)
{
unsafe
{
byte* p = (byte*)data.sharedBodyIndexFrame.Buffer.ToPointer();
if (p != null)
{
byte temp = p[offset];
if (temp != data.selected2DPixelBodyIndex)
{
changed = true;
data.selected2DPixelBodyIndex = temp;
}
}
}
}
}
return changed;
}
private viz.Texture GetTexture(EventTypePluginData data)
{
Debug.Assert(data != null);
viz.Texture value = null;
value = this.UpdateData(data);
return value;
}
