private void HandleEvent(EventType eventType, KStudioEvent eventObj, EventTypePluginData data) { Debug.Assert(data != null); if (eventObj != null) { bool doDataEvent = false; bool doVisibleEvent = false; if ((eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.Depth) || (eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.DepthMonitor)) { lock (this.lockObj) { bool isSelectedData = (this.selectedData == data); HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer(); doVisibleEvent = isSelectedData && (newSharedFrame == null) != (data.sharedDepthFrame == null); data.sharedDepthFrame = newSharedFrame; uint newWidth = nui.Constants.STREAM_DEPTH_WIDTH; uint newHeight = nui.Constants.STREAM_DEPTH_HEIGHT; if ((data.depthMap == null) || (newWidth != data.imageWidth) || (newHeight != data.imageHeight)) { viz.Context context = null; if (this.pluginService != null) { context = this.pluginService.GetContext(eventType); } data.imageWidth = newWidth; data.imageHeight = newHeight; DepthPlugin.CreateTextures(data, context); } if (data.sharedDepthFrame != null) { unsafe { data.depthMap.UpdateData((ushort *)data.sharedDepthFrame.Buffer, data.sharedDepthFrame.Size); } } if (DepthPlugin.UpdateSelectedPixelValue(data)) { doDataEvent = isSelectedData; } } if (doVisibleEvent) { this.RaisePropertyChanged("HasSelected2DPixelData"); } if (doDataEvent) { this.RaisePropertyChanged("Selected2DPixelDepth"); } } } }
public void Render2D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture, float left, float top, float width, float height) { lock (this.lockObj) { nui.Registration registration = null; if (this.pluginService != null) { registration = this.pluginService.GetRegistration(eventType); } switch (eventType) { case EventType.Monitor: BodyPlugin.Render2D(pluginViewSettings, texture, left, top, width, height, this.monitorData, registration); break; case EventType.Inspection: BodyPlugin.Render2D(pluginViewSettings, texture, left, top, width, height, this.inspectionData, registration); break; } } }
public void Render2D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture, float left, float top, float width, float height) { }
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture) { lock (this.lockObj) { EventTypePluginData data = null; nui.Registration registration = null; switch (eventType) { case EventType.Monitor: data = this.monitorData; break; case EventType.Inspection: data = this.inspectionData; break; } if (data != null) { if (this.pluginService != null) { registration = this.pluginService.GetRegistration(eventType); if (data.lastRegistration != registration) { data.lastRegistration = this.pluginService.GetRegistration(eventType); if ((data.lastRegistration != null) && (data.depthMap != null)) { uint xyTableSize; IntPtr xyTable = data.lastRegistration.GetXYTable(out xyTableSize); data.depthMap.UpdateXYTable(xyTable, xyTableSize); } } } DepthPlugin.Render3D(pluginViewSettings, texture, data, registration); } } }
protected override void OnGetLayout(viz.Context context, viz.Texture texture, ref float layoutWidth, ref float layoutHeight) { DebugHelper.AssertUIThread(); bool fixZoom = (this.imageWidth == 0) || (this.imageHeight == 0); if (texture == null) { this.imageWidth = Image2DVisualizationControl.defaultWidth; this.imageHeight = Image2DVisualizationControl.defaultHeight; } else { this.imageWidth = texture.GetWidth(); this.imageHeight = texture.GetHeight(); } double scale = 1.0; if (this.IsZoomToFit) { if (this.scrollViewer != null) { this.scrollViewer.VerticalScrollBarVisibility = ScrollBarVisibility.Hidden; this.scrollViewer.HorizontalScrollBarVisibility = ScrollBarVisibility.Hidden; scale = CalcScaleOnZoomToFit(this.scrollViewer.ActualWidth, this.scrollViewer.ActualHeight); } int zoom = (int)(100.0 * scale); if (this.Zoom != zoom) { if (this.scaleTransform != null) { this.scaleTransform.ScaleX = scale; this.scaleTransform.ScaleY = scale; } fixZoom = true; this.ignoreZoom++; this.SetValue(Image2DVisualizationControl.ZoomProperty, zoom); this.ignoreZoom--; } } else { scale = this.Zoom / 100.0; if (this.scrollViewer != null) { this.scrollViewer.VerticalScrollBarVisibility = ScrollBarVisibility.Auto; this.scrollViewer.HorizontalScrollBarVisibility = ScrollBarVisibility.Auto; } } if (fixZoom) { this.OnZoomChanged(); } layoutWidth = (float)(this.imageWidth * scale); layoutHeight = (float)(this.imageHeight * scale); }
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture) { lock (this.lockObj) { switch (eventType) { case EventType.Monitor: AccessoryPlugin.Render3D(pluginViewSettings, this.monitorAccessory); break; case EventType.Inspection: AccessoryPlugin.Render3D(pluginViewSettings, this.inspectionAccessory); break; } } }
private void HandleEvent(EventType eventType, KStudioEvent eventObj, EventTypePluginData data) { Debug.Assert(data != null); if (eventObj != null) { bool doDataEvent = false; bool doVisibleEvent = false; if (eventObj.EventStreamDataTypeId == KStudioEventStreamDataTypeIds.RawIr) { lock (this.lockObj) { bool isSelectedData = (this.selectedData == data); HGlobalBuffer newSharedFrame = eventObj.GetRetainableEventDataBuffer(); doVisibleEvent = isSelectedData && (newSharedFrame == null) != (data.sharedRawFrame == null); data.sharedRawFrame = newSharedFrame; uint newWidth = nui.Constants.STREAM_IR_WIDTH; uint newHeight = nui.Constants.STREAM_IR_HEIGHT; if ((data.rawIrTexture == null) || (newWidth != data.imageWidth) || (newHeight != data.imageHeight)) { data.imageWidth = newWidth; data.imageHeight = newHeight; viz.Context context = null; if (this.pluginService != null) { context = this.pluginService.GetContext(eventType); } RawIrPlugin.CreateTextures(data, context); } DepthIrEngine depthIrEngine = this.pluginService.DepthIrEngine; if ((depthIrEngine != null) && (data.depthFrame != null) && (data.irFrame != null)) { unsafe { fixed(ushort *pDepthFrame = &data.depthFrame[0], pIrFrame = &data.irFrame[0]) { depthIrEngine.HandleEvent(data.sharedRawFrame.Buffer, data.sharedRawFrame.Size, pDepthFrame, pIrFrame, data.imageWidth, data.imageHeight); uint imageDataSize = data.imageWidth * data.imageHeight * sizeof(ushort); if (data.rawIrTexture != null) { data.rawIrTexture.UpdateData((byte *)pIrFrame, imageDataSize); } if (data.depthMap != null) { data.depthMap.UpdateData(pDepthFrame, imageDataSize); } } } } if (RawIrPlugin.UpdateSelectedPixelValue(data)) { doDataEvent = isSelectedData; } } if (doVisibleEvent) { this.RaisePropertyChanged("HasSelected2DPixelData"); } if (doDataEvent) { this.RaisePropertyChanged("Selected2DPixelIrIntensity"); this.RaisePropertyChanged("Selected2DPixelDepth"); } } } }
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture) { lock (this.lockObj) { nui.Registration registration = null; if (this.pluginService != null) { registration = pluginService.GetRegistration(eventType); } switch (eventType) { case EventType.Monitor: RawIrPlugin.Render3D(pluginViewSettings, texture, this.monitorData, registration); break; case EventType.Inspection: RawIrPlugin.Render3D(pluginViewSettings, texture, this.inspectionData, registration); break; } } }
public void Render3D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture) { if (eventType == EventType.Monitor) { lock (this.lockObj) { if (pluginViewSettings is AudioPlugin3DViewSettings) { if (this.beamConfidence > 0.0f) { viz.Effect effectBeam = new viz.Effect() { EnableLighting = true, Ambient = new viz.Vector(0.0f, 1.0f, 0.0f, 0.5f), }; float[] matrixFloats = new float[16]; unsafe { fixed(float *pMatrix = &matrixFloats[0]) { MatrixHelper.CalculateBeamMatrix(this.beamAngle, pMatrix); } } viz.Matrix mat = new viz.Matrix(); mat.R0 = new viz.Vector(matrixFloats[0], matrixFloats[1], matrixFloats[2], matrixFloats[3]); mat.R1 = new viz.Vector(matrixFloats[4], matrixFloats[5], matrixFloats[6], matrixFloats[7]); mat.R2 = new viz.Vector(matrixFloats[8], matrixFloats[9], matrixFloats[10], matrixFloats[11]); mat.R3 = new viz.Vector(matrixFloats[12], matrixFloats[13], matrixFloats[14], matrixFloats[15]); this.beamMesh.Render(viz.MeshRenderMode.IndexedTriangleList, mat, effectBeam, null); } } } } }
public void Render2D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture, float left, float top, float width, float height) { if (eventType == EventType.Monitor) { lock (this.lockObj) { if (this.frameTime == TimeSpan.MinValue) { return; } AudioPlugin2DViewSettings audioPluginViewSettings = pluginViewSettings as AudioPlugin2DViewSettings; if (audioPluginViewSettings != null) { float x = 10; // margin on left float chartX = x + 100; // offset to chart rendering float y = 10; // margin on top float deltaY = 24; // height of each row // chart window size float chartHeight = 20; float chartWidth = 600; // max value in chart, bigger ones got capped float chartValueCap = 0.02f; // special color for current frame in time line viz.Vector currentFrameColor = new viz::Vector(1, 0, 0, 1); viz.Vector?color = null; if ((audioPluginViewSettings.RenderBeam) && (this.font != null)) { string str; if (this.beamConfidence > 0.0f) { str = string.Format(CultureInfo.CurrentCulture, Strings.Audio_Beam_Label_Format, this.beamAngle); // TODO: option for other display? like degrees? } else { str = Strings.Audio_Beam_Invalid_Label; } this.font.DrawText(str, x, y, color); } y += deltaY; if (this.sharedAudioFrame != null) { if (this.sharedAudioFrame.Size >= cAudioFrameSizeMinimum) { IntPtr bufferPtr = this.sharedAudioFrame.Buffer; unsafe { nui.AUDIO_FRAME * pFrame = (nui.AUDIO_FRAME *)bufferPtr.ToPointer(); nui.AUDIO_SUBFRAME *pSubFrame = &(pFrame->FirstSubFrame); ulong currentFirstFrameTimeStamp = (ulong)this.frameTime.Ticks; ulong timeStampOffset = currentFirstFrameTimeStamp - pSubFrame->TimeCounter; ulong currentLastFrameTimeStamp = pSubFrame[pFrame->SubFrameCount - 1].TimeCounter + timeStampOffset; #if TODO_LOCAL_PLAYBACK if ((this.timelineBegin != 0) || (this.timelineEnd != 0)) { // timeline rendering if (this.timelineDirty) { this.timelineData->Sort(new System.Comparison <TimelineEntry>(TimelineEntry.Compare)); timelineDirty = false; } if (audioPluginViewSettings.RenderOutput) { if (this.font != null) { this.font.DrawText(this.outString, x, y, color); for (int i = 0; i < this.timelineData.Count; i++) { ulong timeStamp = this.timelineData[i]->TimeStamp; if (timeStamp < this.timelineBegin) { continue; } if (timeStamp > this.timelineEnd) { break; } float normalizedValue = (float)(Math.Min(chartValueCap, Math.Abs(this.timelineData[i].Output)) / chartValueCap); float topLeftX = chartX + (timeStamp - this.timelineBegin) * chartWidth / (this.timelineEnd - this.timelineBegin); float topLeftY = y + (1 - normalizedValue) * chartHeight / 2; float chartBarHeight = Math.Max(normalizedValue * chartHeight, 1.0f); bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp); float barZ = isCurrentFrame ? 0 : 0.01f; // put current frame at a smaller z so it's always visible float barWidth = isCurrentFrame ? 2.0f : 1.0f; // make current frame tick bold viz::Vector?barColor = isCurrentFrame ? currentFrameColor : null; if (this.overlay != null) { this.overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor); } } } y += deltaY; } for (int iMIC = 0; iMIC < NUIP_AUDIO_NUM_MIC; iMIC++) { if ((int)renderOptionType == (int)RenderOptionType::MIC0 + iMIC) { _font->DrawText(renderOptionName, x, y, color); for (int iSample = 0; iSample < _timelineData->Count; iSample++) { UInt64 timeStamp = _timelineData[iSample]->TimeStamp; if (timeStamp < _timelineBegin) { continue; } if (timeStamp > _timelineEnd) { break; } float normalizedValue = min(chartValueCap, fabs(_timelineData[iSample]->MIC[iMIC])) / chartValueCap; float topLeftX = chartX + (timeStamp - _timelineBegin) * chartWidth / (_timelineEnd - _timelineBegin); float topLeftY = y + (1 - normalizedValue) * chartHeight / 2; float chartBarHeight = max(normalizedValue * chartHeight, 1.0f); bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp); float barZ = isCurrentFrame ? 0 : 0.01f; float barWidth = isCurrentFrame ? 2.0f : 1.0f; Nullable <Xbox::Kinect::Viz::Vector> barColor; if (isCurrentFrame) { barColor = currentFrameColor; } _overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor); } } y += deltaY; } for (int iSPK = 0; iSPK < NUIP_AUDIO_NUM_SPK; iSPK++) { if ((int)renderOptionType == (int)RenderOptionType::SPK0 + iSPK) { _font->DrawText(renderOptionName, x, y, color); for (int iSample = 0; iSample < _timelineData->Count; iSample++) { UInt64 timeStamp = _timelineData[iSample]->TimeStamp; if (timeStamp < _timelineBegin) { continue; } if (timeStamp > _timelineEnd) { break; } float normalizedValue = min(chartValueCap, fabs(_timelineData[iSample]->SPK[iSPK])) / chartValueCap; float topLeftX = chartX + (timeStamp - _timelineBegin) * chartWidth / (_timelineEnd - _timelineBegin); float topLeftY = y + (1 - normalizedValue) * chartHeight / 2; float chartBarHeight = max(normalizedValue * chartHeight, 1.0f); bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp); float barZ = isCurrentFrame ? 0 : 0.01f; float barWidth = isCurrentFrame ? 2.0f : 1.0f; Nullable <Xbox::Kinect::Viz::Vector> barColor; if (isCurrentFrame) { barColor = currentFrameColor; } _overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor); } } y += deltaY; } } else #endif // TODO_LOCAL_PLAYBACK { // live rendering float barWidth = 1; UInt64 timeSpan = 2 * 10 * 1000 * 1000; // in 100ns unit if (audioPluginViewSettings.RenderOutput) { if (this.font != null) { this.font.DrawText(this.outString, x, y, color); } if (this.outChart != null) { this.outChart.RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap); } y += deltaY; } if (this.micCharts != null) { for (int i = 0; i < this.micCharts.Length; ++i) { if (audioPluginViewSettings.GetTrackOption((AudioTrack)(AudioTrack.Mic0 + i))) { if (this.font != null) { this.font.DrawText(this.micStrings[i], x, y, color); } this.micCharts[i].RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap); y += deltaY; } } } if (this.speakerCharts != null) { for (int i = 0; i < this.speakerCharts.Length; ++i) { if (audioPluginViewSettings.GetTrackOption((AudioTrack)(AudioTrack.SpeakerL + i))) { if (this.font != null) { this.font.DrawText(this.speakerStrings[i], x, y, color); } this.speakerCharts[i].RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap); y += deltaY; } } } } } } } } } } }
public void InitializeRender(EventType eventType, viz.Context context) { if (eventType == EventType.Monitor) { lock (this.lockObj) { if (this.audioPlayer == null) { try { this.audioPlayer = new AudioPlayer(); } catch (Exception) { if (this.loggingService != null) { this.loggingService.LogLine(Strings.Audio_Error_CannotInitializePlayer); } } } if (context != null) { this.font = new viz.Font(context); this.overlay = new viz.Overlay(context); const float cHorizontalFov = 70.6f; float beamZ = 4.5f; // max depth in meters float beamY = (float)(beamZ * Math.Tan(cHorizontalFov / 2.0)); viz.Vertex[] vertices = new viz.Vertex[] { new viz.Vertex(0, 0, 0, 0, 0, 0, 0, 0, 0xFFFFFFFF), new viz.Vertex(0, beamY, beamZ, 0, 0, 0, 0, 0, 0xFFFFFFFF), new viz.Vertex(0, -beamY, beamZ, 0, 0, 0, 0, 0, 0xFFFFFFFF), }; uint[] indices = new uint[] { 0, 1, 2, 0, 2, 1, }; this.beamMesh = new viz::Mesh(context, (uint)vertices.Length, (uint)indices.Length); this.beamMesh.UpdateVertex(vertices); this.beamMesh.UpdateIndex(indices); this.outChart = new viz.TemporalChart(context); this.micCharts = new viz.TemporalChart[nui.Constants.AUDIO_NUM_MIC]; for (int i = 0; i < this.micCharts.Length; ++i) { this.micCharts[i] = new viz.TemporalChart(context); } this.speakerCharts = new viz.TemporalChart[nui.Constants.AUDIO_NUM_SPK]; for (int i = 0; i < this.speakerCharts.Length; ++i) { this.speakerCharts[i] = new viz.TemporalChart(context); } #if TODO_AUDIO_OUT // audio out if (FAILED(InitializeWASAPI())) { // TODO_LOG } #endif // TODO_AUDIO_OUT } } } }