public void HandleEvent(EventType eventType, KStudioEvent eventObj) { if ((eventType == EventType.Monitor) && (eventObj != null)) { if ((eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.SystemAudio) || (eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.SystemAudioMonitor) || (eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.TitleAudio) || (eventObj.EventStreamDataTypeId == HackKStudioEventStreamDataTypeIds.TitleAudioMonitor)) { lock (this.lockObj) { this.sharedAudioFrame = null; this.beamConfidence = 0.0f; this.beamAngle = 0.0f; this.frameTime = TimeSpan.Zero; uint bufferSize; IntPtr bufferPtr; eventObj.AccessUnderlyingEventDataBuffer(out bufferSize, out bufferPtr); if (bufferSize >= cAudioFrameSizeMinimum) { this.sharedAudioFrame = eventObj.GetRetainableEventDataBuffer(); this.frameTime = eventObj.RelativeTime; unsafe { nui.AUDIO_FRAME * pFrame = (nui.AUDIO_FRAME *)bufferPtr.ToPointer(); nui.AUDIO_SUBFRAME * pSubFrame = &(pFrame->FirstSubFrame); nui.AUDIO_BEAM_FRAME_HEADER *pBeamHeader = &(pSubFrame->BeamFrameHeader); this.beamConfidence = pBeamHeader->BeamAngleConfidence; this.beamAngle = pBeamHeader->BeamAngle; for (int i = 0; i < pFrame->SubFrameCount; ++i) { UpdateChartSubFrame(pSubFrame + i); } } } } } } }
unsafe private void UpdateChartSubFrame(nui.AUDIO_SUBFRAME *pSubFrame) { if (this.outChart != null) { IntPtr ptrOut = new IntPtr(pSubFrame) + cOutOffset; float *pFloats = (float *)ptrOut; float average = 0.0f; for (uint i = 0; i < nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME; ++i) { average += Math.Abs(pFloats[i]); } average = average / nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME; this.outChart.Update(average, pSubFrame->TimeCounter); } if (this.micCharts != null) { float[] averages = new float[this.micCharts.Length]; IntPtr ptrMic = new IntPtr(pSubFrame) + cMicOffset; float * pFloats = (float *)ptrMic; #if OLD // less performant fixed(float *pAverages = &averages[0]) { AverageHelper.CalculateAbsAverageBy4((uint)nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME * 4, pFloats, pAverages); } #endif // OLD uint total = nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME * 4; for (uint i = 0; i < total; i += 4) { averages[0] += Math.Abs(pFloats[i]); averages[1] += Math.Abs(pFloats[i + 1]); averages[2] += Math.Abs(pFloats[i + 2]); averages[3] += Math.Abs(pFloats[i + 3]); } for (int i = 0; i < 4; ++i) { averages[i] /= nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME; this.micCharts[i].Update(averages[i], pSubFrame->TimeCounter); } } if (this.speakerCharts != null) { float[] averages = new float[this.speakerCharts.Length]; IntPtr ptrSpeaker = new IntPtr(pSubFrame) + cSpeakerOffset; float * pFloats = (float *)ptrSpeaker; #if OLD fixed(float *pAverages = &averages[0]) { AverageHelper.CalculateAbsAverageBy8((uint)nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME * 8, pFloats, pAverages); } #endif // OLD uint total = nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME * 8; for (uint i = 0; i < total; i += 8) { averages[0] += Math.Abs(pFloats[i]); averages[1] += Math.Abs(pFloats[i + 1]); averages[2] += Math.Abs(pFloats[i + 2]); averages[3] += Math.Abs(pFloats[i + 3]); averages[4] += Math.Abs(pFloats[i + 4]); averages[5] += Math.Abs(pFloats[i + 5]); averages[6] += Math.Abs(pFloats[i + 6]); averages[7] += Math.Abs(pFloats[i + 7]); } for (int i = 0; i < this.speakerCharts.Length; ++i) { averages[i] /= nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME; this.speakerCharts[i].Update(averages[i], pSubFrame->TimeCounter); } } if (this.IsAudible && (this.audioPlayer != null)) { int offset = 0; uint stride = 0; switch (this.AudibleTrack) { case AudioTrack.Output: stride = 1; offset = AudioPlugin.cOutOffset; break; case AudioTrack.Mic0: case AudioTrack.Mic1: case AudioTrack.Mic2: case AudioTrack.Mic3: stride = (uint)this.micCharts.Length; offset = AudioPlugin.cMicOffset + (sizeof(float) * (int)(this.AudibleTrack - AudioTrack.Mic0)); break; case AudioTrack.SpeakerL: case AudioTrack.SpeakerR: case AudioTrack.SpeakerC: case AudioTrack.SpeakerLFE: case AudioTrack.SpeakerBL: case AudioTrack.SpeakerBR: case AudioTrack.SpeakerSL: case AudioTrack.SpeakerSR: stride = (uint)this.speakerCharts.Length; offset = AudioPlugin.cSpeakerOffset + (sizeof(float) * (int)(this.AudibleTrack - AudioTrack.SpeakerL)); break; } if (stride != 0) { IntPtr ptr = new IntPtr(pSubFrame) + offset; float *pFloats = (float *)ptr; this.audioPlayer.PlayAudio(nui.Constants.AUDIO_SAMPLES_PER_SUBFRAME, pFloats, nui.Constants.AUDIO_SAMPLERATE, stride); } } }
public void Render2D(EventType eventType, IPluginViewSettings pluginViewSettings, viz.Context context, viz.Texture texture, float left, float top, float width, float height) { if (eventType == EventType.Monitor) { lock (this.lockObj) { if (this.frameTime == TimeSpan.MinValue) { return; } AudioPlugin2DViewSettings audioPluginViewSettings = pluginViewSettings as AudioPlugin2DViewSettings; if (audioPluginViewSettings != null) { float x = 10; // margin on left float chartX = x + 100; // offset to chart rendering float y = 10; // margin on top float deltaY = 24; // height of each row // chart window size float chartHeight = 20; float chartWidth = 600; // max value in chart, bigger ones got capped float chartValueCap = 0.02f; // special color for current frame in time line viz.Vector currentFrameColor = new viz::Vector(1, 0, 0, 1); viz.Vector?color = null; if ((audioPluginViewSettings.RenderBeam) && (this.font != null)) { string str; if (this.beamConfidence > 0.0f) { str = string.Format(CultureInfo.CurrentCulture, Strings.Audio_Beam_Label_Format, this.beamAngle); // TODO: option for other display? like degrees? } else { str = Strings.Audio_Beam_Invalid_Label; } this.font.DrawText(str, x, y, color); } y += deltaY; if (this.sharedAudioFrame != null) { if (this.sharedAudioFrame.Size >= cAudioFrameSizeMinimum) { IntPtr bufferPtr = this.sharedAudioFrame.Buffer; unsafe { nui.AUDIO_FRAME * pFrame = (nui.AUDIO_FRAME *)bufferPtr.ToPointer(); nui.AUDIO_SUBFRAME *pSubFrame = &(pFrame->FirstSubFrame); ulong currentFirstFrameTimeStamp = (ulong)this.frameTime.Ticks; ulong timeStampOffset = currentFirstFrameTimeStamp - pSubFrame->TimeCounter; ulong currentLastFrameTimeStamp = pSubFrame[pFrame->SubFrameCount - 1].TimeCounter + timeStampOffset; #if TODO_LOCAL_PLAYBACK if ((this.timelineBegin != 0) || (this.timelineEnd != 0)) { // timeline rendering if (this.timelineDirty) { this.timelineData->Sort(new System.Comparison <TimelineEntry>(TimelineEntry.Compare)); timelineDirty = false; } if (audioPluginViewSettings.RenderOutput) { if (this.font != null) { this.font.DrawText(this.outString, x, y, color); for (int i = 0; i < this.timelineData.Count; i++) { ulong timeStamp = this.timelineData[i]->TimeStamp; if (timeStamp < this.timelineBegin) { continue; } if (timeStamp > this.timelineEnd) { break; } float normalizedValue = (float)(Math.Min(chartValueCap, Math.Abs(this.timelineData[i].Output)) / chartValueCap); float topLeftX = chartX + (timeStamp - this.timelineBegin) * chartWidth / (this.timelineEnd - this.timelineBegin); float topLeftY = y + (1 - normalizedValue) * chartHeight / 2; float chartBarHeight = Math.Max(normalizedValue * chartHeight, 1.0f); bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp); float barZ = isCurrentFrame ? 0 : 0.01f; // put current frame at a smaller z so it's always visible float barWidth = isCurrentFrame ? 2.0f : 1.0f; // make current frame tick bold viz::Vector?barColor = isCurrentFrame ? currentFrameColor : null; if (this.overlay != null) { this.overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor); } } } y += deltaY; } for (int iMIC = 0; iMIC < NUIP_AUDIO_NUM_MIC; iMIC++) { if ((int)renderOptionType == (int)RenderOptionType::MIC0 + iMIC) { _font->DrawText(renderOptionName, x, y, color); for (int iSample = 0; iSample < _timelineData->Count; iSample++) { UInt64 timeStamp = _timelineData[iSample]->TimeStamp; if (timeStamp < _timelineBegin) { continue; } if (timeStamp > _timelineEnd) { break; } float normalizedValue = min(chartValueCap, fabs(_timelineData[iSample]->MIC[iMIC])) / chartValueCap; float topLeftX = chartX + (timeStamp - _timelineBegin) * chartWidth / (_timelineEnd - _timelineBegin); float topLeftY = y + (1 - normalizedValue) * chartHeight / 2; float chartBarHeight = max(normalizedValue * chartHeight, 1.0f); bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp); float barZ = isCurrentFrame ? 0 : 0.01f; float barWidth = isCurrentFrame ? 2.0f : 1.0f; Nullable <Xbox::Kinect::Viz::Vector> barColor; if (isCurrentFrame) { barColor = currentFrameColor; } _overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor); } } y += deltaY; } for (int iSPK = 0; iSPK < NUIP_AUDIO_NUM_SPK; iSPK++) { if ((int)renderOptionType == (int)RenderOptionType::SPK0 + iSPK) { _font->DrawText(renderOptionName, x, y, color); for (int iSample = 0; iSample < _timelineData->Count; iSample++) { UInt64 timeStamp = _timelineData[iSample]->TimeStamp; if (timeStamp < _timelineBegin) { continue; } if (timeStamp > _timelineEnd) { break; } float normalizedValue = min(chartValueCap, fabs(_timelineData[iSample]->SPK[iSPK])) / chartValueCap; float topLeftX = chartX + (timeStamp - _timelineBegin) * chartWidth / (_timelineEnd - _timelineBegin); float topLeftY = y + (1 - normalizedValue) * chartHeight / 2; float chartBarHeight = max(normalizedValue * chartHeight, 1.0f); bool isCurrentFrame = (timeStamp >= currentFirstFrameTimeStamp && timeStamp <= currentLastFrameTimeStamp); float barZ = isCurrentFrame ? 0 : 0.01f; float barWidth = isCurrentFrame ? 2.0f : 1.0f; Nullable <Xbox::Kinect::Viz::Vector> barColor; if (isCurrentFrame) { barColor = currentFrameColor; } _overlay->DrawColor(topLeftX, topLeftY, barWidth, chartBarHeight, barZ, barColor); } } y += deltaY; } } else #endif // TODO_LOCAL_PLAYBACK { // live rendering float barWidth = 1; UInt64 timeSpan = 2 * 10 * 1000 * 1000; // in 100ns unit if (audioPluginViewSettings.RenderOutput) { if (this.font != null) { this.font.DrawText(this.outString, x, y, color); } if (this.outChart != null) { this.outChart.RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap); } y += deltaY; } if (this.micCharts != null) { for (int i = 0; i < this.micCharts.Length; ++i) { if (audioPluginViewSettings.GetTrackOption((AudioTrack)(AudioTrack.Mic0 + i))) { if (this.font != null) { this.font.DrawText(this.micStrings[i], x, y, color); } this.micCharts[i].RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap); y += deltaY; } } } if (this.speakerCharts != null) { for (int i = 0; i < this.speakerCharts.Length; ++i) { if (audioPluginViewSettings.GetTrackOption((AudioTrack)(AudioTrack.SpeakerL + i))) { if (this.font != null) { this.font.DrawText(this.speakerStrings[i], x, y, color); } this.speakerCharts[i].RenderBar(chartX, y, chartWidth, chartHeight, 0, barWidth, color, currentLastFrameTimeStamp, timeSpan, 0, chartValueCap); y += deltaY; } } } } } } } } } } }