private void HandleEvent(EventType eventType, KStudioEvent eventObj, EventTypePluginData data)
{
Debug.Assert(data != null);
if (eventObj != null)
{
bool doDataEvent = false;
bool doVisibleEvent = false;
if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.Depth) ||
(eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.DepthMonitor))
{
lock (this.lockObj)
{
bool isSelectedData = (this.selectedData == data);
HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer();
doVisibleEvent = isSelectedData && (newSharedFrame == null) != (data.sharedDepthFrame == null);
data.sharedDepthFrame = newSharedFrame;
uint newWidth = nui.Constants.STREAM_DEPTH_WIDTH;
uint newHeight = nui.Constants.STREAM_DEPTH_HEIGHT;
if ((data.depthMap == null) || (newWidth != data.imageWidth) || (newHeight != data.imageHeight))
{
viz.Context context = null;
if (this.pluginService != null)
{
context = this.pluginService.GetContext(eventType);
}
data.imageWidth = newWidth;
data.imageHeight = newHeight;
DepthPlugin.CreateTextures(data, context);
}
if (data.sharedDepthFrame != null)
{
unsafe
{
data.depthMap.UpdateData((ushort *)data.sharedDepthFrame.Buffer, data.sharedDepthFrame.Size);
}
}
if (DepthPlugin.UpdateSelectedPixelValue(data))
{
doDataEvent = isSelectedData;
}
}
if (doVisibleEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
if (doDataEvent)
{
this.RaisePropertyChanged("Selected2DPixelDepth");
}
}
}
}
public void Render2D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture, float left, float top, float width, float height)
{
lock (this.lockObj)
{
nui.Registration registration = null;
if (this.pluginService != null)
{
registration = this.pluginService.GetRegistration(eventType);
}
switch (eventType)
{
case EventType.Monitor:
BodyPlugin.Render2D(pluginViewSettings, texture, left, top, width, height, this.monitorData, registration);
break;
case EventType.Inspection:
BodyPlugin.Render2D(pluginViewSettings, texture, left, top, width, height, this.inspectionData, registration);
break;
}
}
}
public void Render2D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture, float left, float top, float width, float height)
{
}
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture)
{
lock (this.lockObj)
{
EventTypePluginData data = null;
nui.Registration registration = null;
switch (eventType)
{
case EventType.Monitor:
data = this.monitorData;
break;
case EventType.Inspection:
data = this.inspectionData;
break;
}
if (data != null)
{
if (this.pluginService != null)
{
registration = this.pluginService.GetRegistration(eventType);
if (data.lastRegistration != registration)
{
data.lastRegistration = this.pluginService.GetRegistration(eventType);
if ((data.lastRegistration != null) && (data.depthMap != null))
{
uint xyTableSize;
IntPtr xyTable = data.lastRegistration.GetXYTable(out xyTableSize);
data.depthMap.UpdateXYTable(xyTable, xyTableSize);
}
}
}
DepthPlugin.Render3D(pluginViewSettings, texture, data, registration);
}
}
}
protected override void OnGetLayout(viz.Context context, viz.Texture texture, ref float layoutWidth, ref float layoutHeight)
{
DebugHelper.AssertUIThread();
bool fixZoom = (this.imageWidth == 0) || (this.imageHeight == 0);
if (texture == null)
{
this.imageWidth = Image2DVisualizationControl.defaultWidth;
this.imageHeight = Image2DVisualizationControl.defaultHeight;
}
else
{
this.imageWidth = texture.GetWidth();
this.imageHeight = texture.GetHeight();
}
double scale = 1.0;
if (this.IsZoomToFit)
{
if (this.scrollViewer != null)
{
this.scrollViewer.VerticalScrollBarVisibility = ScrollBarVisibility.Hidden;
this.scrollViewer.HorizontalScrollBarVisibility = ScrollBarVisibility.Hidden;
scale = CalcScaleOnZoomToFit(this.scrollViewer.ActualWidth, this.scrollViewer.ActualHeight);
}
int zoom = (int)(100.0 * scale);
if (this.Zoom != zoom)
{
if (this.scaleTransform != null)
{
this.scaleTransform.ScaleX = scale;
this.scaleTransform.ScaleY = scale;
}
fixZoom = true;
this.ignoreZoom++;
this.SetValue(Image2DVisualizationControl.ZoomProperty, zoom);
this.ignoreZoom--;
}
}
else
{
scale = this.Zoom / 100.0;
if (this.scrollViewer != null)
{
this.scrollViewer.VerticalScrollBarVisibility = ScrollBarVisibility.Auto;
this.scrollViewer.HorizontalScrollBarVisibility = ScrollBarVisibility.Auto;
}
}
if (fixZoom)
{
this.OnZoomChanged();
}
layoutWidth = (float)(this.imageWidth * scale);
layoutHeight = (float)(this.imageHeight * scale);
}
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture)
{
lock (this.lockObj)
{
switch (eventType)
{
case EventType.Monitor:
AccessoryPlugin.Render3D(pluginViewSettings, this.monitorAccessory);
break;
case EventType.Inspection:
AccessoryPlugin.Render3D(pluginViewSettings, this.inspectionAccessory);
break;
}
}
}
private void HandleEvent(EventType eventType, KStudioEvent eventObj, EventTypePluginData data)
{
Debug.Assert(data != null);
if (eventObj != null)
{
bool doDataEvent = false;
bool doVisibleEvent = false;
if (eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.RawIr)
{
lock (this.lockObj)
{
bool isSelectedData = (this.selectedData == data);
HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer();
doVisibleEvent = isSelectedData && (newSharedFrame == null) != (data.sharedRawFrame == null);
data.sharedRawFrame = newSharedFrame;
uint newWidth = nui.Constants.STREAM_IR_WIDTH;
uint newHeight = nui.Constants.STREAM_IR_HEIGHT;
if ((data.rawIrTexture == null) || (newWidth != data.imageWidth) || (newHeight != data.imageHeight))
{
data.imageWidth = newWidth;
data.imageHeight = newHeight;
viz.Context context = null;
if (this.pluginService != null)
{
context = this.pluginService.GetContext(eventType);
}
RawIrPlugin.CreateTextures(data, context);
}
DepthIrEngine depthIrEngine = this.pluginService.DepthIrEngine;
if ((depthIrEngine != null) && (data.depthFrame != null) && (data.irFrame != null))
{
unsafe
{
fixed(ushort *pDepthFrame = &data.depthFrame[0], pIrFrame = &data.irFrame[0])
{
depthIrEngine.HandleEvent(data.sharedRawFrame.Buffer, data.sharedRawFrame.Size, pDepthFrame, pIrFrame, data.imageWidth, data.imageHeight);
uint imageDataSize = data.imageWidth * data.imageHeight * sizeof(ushort);
if (data.rawIrTexture != null)
{
data.rawIrTexture.UpdateData((byte *)pIrFrame, imageDataSize);
}
if (data.depthMap != null)
{
data.depthMap.UpdateData(pDepthFrame, imageDataSize);
}
}
}
}
if (RawIrPlugin.UpdateSelectedPixelValue(data))
{
doDataEvent = isSelectedData;
}
}
if (doVisibleEvent)
{
this.RaisePropertyChanged("HasSelected2DPixelData");
}
if (doDataEvent)
{
this.RaisePropertyChanged("Selected2DPixelIrIntensity");
this.RaisePropertyChanged("Selected2DPixelDepth");
}
}
}
}
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture)
{
lock (this.lockObj)
{
nui.Registration registration = null;
if (this.pluginService != null)
{
registration = pluginService.GetRegistration(eventType);
}
switch (eventType)
{
case EventType.Monitor:
RawIrPlugin.Render3D(pluginViewSettings, texture, this.monitorData, registration);
break;
case EventType.Inspection:
RawIrPlugin.Render3D(pluginViewSettings, texture, this.inspectionData, registration);
break;
}
}
}
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture)
{
if (eventType == EventType.Monitor)
{
lock (this.lockObj)
{
if (pluginViewSettings is AudioPlugin3DViewSettings)
{
if (this.beamConfidence > 0.0f)
{
viz.Effect effectBeam = new viz.Effect()
{
EnableLighting = true,
Ambient = new viz.Vector(0.0f, 1.0f, 0.0f, 0.5f),
};
float[] matrixFloats = new float[16];
unsafe
{
fixed(float *pMatrix = &matrixFloats[0])
{
MatrixHelper.CalculateBeamMatrix(this.beamAngle, pMatrix);
}
}
viz.Matrix mat = new viz.Matrix();
mat.R0 = new viz.Vector(matrixFloats[0], matrixFloats[1], matrixFloats[2], matrixFloats[3]);
mat.R1 = new viz.Vector(matrixFloats[4], matrixFloats[5], matrixFloats[6], matrixFloats[7]);
mat.R2 = new viz.Vector(matrixFloats[8], matrixFloats[9], matrixFloats[10], matrixFloats[11]);
mat.R3 = new viz.Vector(matrixFloats[12], matrixFloats[13], matrixFloats[14], matrixFloats[15]);
this.beamMesh.Render(viz.MeshRenderMode.IndexedTriangleList, mat, effectBeam, null);
}
}
}
}
}
public void Render2D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture, float left, float top, float width, float height)
{
if (eventType == EventType.Monitor)
{
lock (this.lockObj)
{
if (this.frameTime == TimeSpan.MinValue)
{
return;
}
AudioPlugin2DViewSettings audioPluginViewSettings = pluginViewSettings as AudioPlugin2DViewSettings;
if (audioPluginViewSettings != null)
{
float x = 10; // margin on left
float chartX = x + 100; // offset to chart rendering
float y = 10; // margin on top
float deltaY = 24; // height of each row
// chart window size
float chartHeight = 20;
float chartWidth = 600;
// max value in chart, bigger ones got capped
float chartValueCap = 0.02f;
// special color for current frame in time line
viz.Vector currentFrameColor = new viz::Vector(1, 0, 0, 1);
viz.Vector?color = null;
if ((audioPluginViewSettings.RenderBeam) && (this.font != null))
{
string str;
if (this.beamConfidence > 0.0f)
{
str = string.Format(CultureInfo.CurrentCulture, Strings.Audio_Beam_Label_Format, this.beamAngle); // TODO: option for other display? like degrees?
}
else
{
str = Strings.Audio_Beam_Invalid_Label;
}
this.font.DrawText(str, x, y, color);
}
y += deltaY;
if (this.sharedAudioFrame != null)
{
if (this.sharedAudioFrame.Size >= cAudioFrameSizeMinimum)
{
IntPtr bufferPtr = this.sharedAudioFrame.Buffer;
unsafe
{
nui.AUDIO_FRAME * pFrame = (nui.AUDIO_FRAME *)bufferPtr.ToPointer();
nui.AUDIO_SUBFRAME *pSubFrame = &(pFrame->FirstSubFrame);
ulong currentFirstFrameTimeStamp = (ulong)this.frameTime.Ticks;
ulong timeStampOffset = currentFirstFrameTimeStamp - pSubFrame->TimeCounter;
ulong currentLastFrameTimeStamp = pSubFrame[pFrame->SubFrameCount - 1].TimeCounter + timeStampOffset;
#if TODO_LOCAL_PLAYBACK
if ((this.timelineBegin != 0) || (this.timelineEnd != 0))
{
// timeline rendering
if (this.timelineDirty)
{
this.timelineData->Sort(new System.Comparison <TimelineEntry>(TimelineEntry.Compare));
timelineDirty = false;
}
if (audioPluginViewSettings.RenderOutput)
{
if (this.font != null)
{
this.font.DrawText(this.outString, x, y, color);
for (int i = 0; i < this.timelineData.Count; i++)
{
ulong timeStamp = this.timelineData[i]->TimeStamp;
if (timeStamp < this.timelineBegin)
{
continue;
}
if (timeStamp > this.timelineEnd)
{
break;
}
float normalizedValue = (float)(Math.Min(chartValueCap, Math.Abs(this.timelineData[i].Output)) / chartValueCap);
float topLeftX = chartX + (timeStamp - this.timelineBegin) * chartWidth / (this.timelineEnd - this.timelineBegin);
float topLeftY = y + (1 - normalizedValue) * chartHeight / 2;
float chartBarHeight = Math.Max(normalizedValue * chartHeight, 1.0f);
bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp);
float barZ = isCurrentFrame ? 0 : 0.01f; // put current frame at a smaller z so it's always visible
float barWidth = isCurrentFrame ? 2.0f : 1.0f; // make current frame tick bold
viz::Vector?barColor = isCurrentFrame ? currentFrameColor : null;
if (this.overlay != null)
{
this.overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor);
}
}
}
y += deltaY;
}
for (int iMIC = 0; iMIC < NUIP_AUDIO_NUM_MIC; iMIC++)
{
if ((int)renderOptionType == (int)RenderOptionType::MIC0 + iMIC)
{
_font->DrawText(renderOptionName, x, y, color);
for (int iSample = 0; iSample < _timelineData->Count; iSample++)
{
UInt64 timeStamp = _timelineData[iSample]->TimeStamp;
if (timeStamp < _timelineBegin)
{
continue;
}
if (timeStamp > _timelineEnd)
{
break;
}
float normalizedValue = min(chartValueCap, fabs(_timelineData[iSample]->MIC[iMIC])) / chartValueCap;
float topLeftX = chartX + (timeStamp - _timelineBegin) * chartWidth / (_timelineEnd - _timelineBegin);
float topLeftY = y + (1 - normalizedValue) * chartHeight / 2;
float chartBarHeight = max(normalizedValue * chartHeight, 1.0f);
bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp);
float barZ = isCurrentFrame ? 0 : 0.01f;
float barWidth = isCurrentFrame ? 2.0f : 1.0f;
Nullable <Xbox::Kinect::Viz::Vector> barColor;
if (isCurrentFrame)
{
barColor = currentFrameColor;
}
_overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor);
}
}
y += deltaY;
}
for (int iSPK = 0; iSPK < NUIP_AUDIO_NUM_SPK; iSPK++)
{
if ((int)renderOptionType == (int)RenderOptionType::SPK0 + iSPK)
{
_font->DrawText(renderOptionName, x, y, color);
for (int iSample = 0; iSample < _timelineData->Count; iSample++)
{
UInt64 timeStamp = _timelineData[iSample]->TimeStamp;
if (timeStamp < _timelineBegin)
{
continue;
}
if (timeStamp > _timelineEnd)
{
break;
}
float normalizedValue = min(chartValueCap, fabs(_timelineData[iSample]->SPK[iSPK])) / chartValueCap;
float topLeftX = chartX + (timeStamp - _timelineBegin) * chartWidth / (_timelineEnd - _timelineBegin);
float topLeftY = y + (1 - normalizedValue) * chartHeight / 2;
float chartBarHeight = max(normalizedValue * chartHeight, 1.0f);
bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp);
float barZ = isCurrentFrame ? 0 : 0.01f;
float barWidth = isCurrentFrame ? 2.0f : 1.0f;
Nullable <Xbox::Kinect::Viz::Vector> barColor;
if (isCurrentFrame)
{
barColor = currentFrameColor;
}
_overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor);
}
}
y += deltaY;
}
}
else
#endif // TODO_LOCAL_PLAYBACK
{
// live rendering
float barWidth = 1;
UInt64 timeSpan = 2 * 10 * 1000 * 1000; // in 100ns unit
if (audioPluginViewSettings.RenderOutput)
{
if (this.font != null)
{
this.font.DrawText(this.outString, x, y, color);
}
if (this.outChart != null)
{
this.outChart.RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap);
}
y += deltaY;
}
if (this.micCharts != null)
{
for (int i = 0; i < this.micCharts.Length; ++i)
{
if (audioPluginViewSettings.GetTrackOption((AudioTrack)(AudioTrack.Mic0 + i)))
{
if (this.font != null)
{
this.font.DrawText(this.micStrings[i], x, y, color);
}
this.micCharts[i].RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap);
y += deltaY;
}
}
}
if (this.speakerCharts != null)
{
for (int i = 0; i < this.speakerCharts.Length; ++i)
{
if (audioPluginViewSettings.GetTrackOption((AudioTrack)(AudioTrack.SpeakerL + i)))
{
if (this.font != null)
{
this.font.DrawText(this.speakerStrings[i], x, y, color);
}
this.speakerCharts[i].RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap);
y += deltaY;
}
}
}
}
}
}
}
}
}
}
}
public void InitializeRender(EventType eventType, viz.Context context)
{
if (eventType == EventType.Monitor)
{
lock (this.lockObj)
{
if (this.audioPlayer == null)
{
try
{
this.audioPlayer = new AudioPlayer();
}
catch (Exception)
{
if (this.loggingService != null)
{
this.loggingService.LogLine(Strings.Audio_Error_CannotInitializePlayer);
}
}
}
if (context != null)
{
this.font = new viz.Font(context);
this.overlay = new viz.Overlay(context);
const float cHorizontalFov = 70.6f;
float beamZ = 4.5f; // max depth in meters
float beamY = (float)(beamZ * Math.Tan(cHorizontalFov / 2.0));
viz.Vertex[] vertices = new viz.Vertex[]
{
new viz.Vertex(0, 0, 0, 0, 0, 0, 0, 0, 0xFFFFFFFF),
new viz.Vertex(0, beamY, beamZ, 0, 0, 0, 0, 0, 0xFFFFFFFF),
new viz.Vertex(0, -beamY, beamZ, 0, 0, 0, 0, 0, 0xFFFFFFFF),
};
uint[] indices = new uint[]
{
0, 1, 2, 0, 2, 1,
};
this.beamMesh = new viz::Mesh(context, (uint)vertices.Length, (uint)indices.Length);
this.beamMesh.UpdateVertex(vertices);
this.beamMesh.UpdateIndex(indices);
this.outChart = new viz.TemporalChart(context);
this.micCharts = new viz.TemporalChart[nui.Constants.AUDIO_NUM_MIC];
for (int i = 0; i < this.micCharts.Length; ++i)
{
this.micCharts[i] = new viz.TemporalChart(context);
}
this.speakerCharts = new viz.TemporalChart[nui.Constants.AUDIO_NUM_SPK];
for (int i = 0; i < this.speakerCharts.Length; ++i)
{
this.speakerCharts[i] = new viz.TemporalChart(context);
}
#if TODO_AUDIO_OUT // audio out
if (FAILED(InitializeWASAPI()))
{
// TODO_LOG
}
#endif // TODO_AUDIO_OUT
}
}
}
}
