/// <summary> /// /// </summary> /// <param name="inputFormat">format of audio supplied to `Send`</param> /// <param name="inputFrameSize">Size of frames which will should be provided from `GetFrameBuffer`</param> /// <param name="intermediateFrameSize">Size of frames which should be passed into `PreprocessAudioFrame`</param> /// <param name="intermediateSampleRate">Sample rate which should be passed into `PreprocessAudioFrame`</param> /// <param name="outputFrameSize">Size of frames which will be provided sent to `SendSamplesToSubscribers`</param> /// <param name="outputSampleRate"></param> protected BasePreprocessingPipeline(WaveFormat inputFormat, int inputFrameSize, int intermediateFrameSize, int intermediateSampleRate, int outputFrameSize, int outputSampleRate) { if (inputFrameSize < 0) { throw new ArgumentOutOfRangeException("inputFrameSize", "Input frame size cannot be less than zero"); } _inputFrameSize = inputFrameSize; _outputFrameSize = outputFrameSize; _outputFormat = new WaveFormat(1, outputSampleRate); //Create input system (source of empty buffers, queue of waiting input data) _inputBufferSource = new ConcurrentPool <float[]>(24, () => new float[inputFrameSize]); _inputQueue = new TransferBuffer <float[]>(12); _emptyInputFrame = new float[inputFrameSize]; //Create resampler to resample input to intermediate rate _resamplerInput = new BufferedSampleProvider(inputFormat, InputFrameSize * 4); _resampler = new Resampler(_resamplerInput, 48000); _resampledOutput = new SampleToFrameProvider(_resampler, (uint)OutputFrameSize); _intermediateFrame = new float[intermediateFrameSize]; _threadEvent = new AutoResetEvent(false); _thread = new DThread(ThreadEntry); }
public EncoderPipeline([NotNull] WaveFormat inputFormat, [NotNull] IVoiceEncoder encoder, [NotNull] ICommsNetwork net) { if (inputFormat == null) { throw new ArgumentNullException("inputFormat"); } if (encoder == null) { throw new ArgumentNullException("encoder"); } if (net == null) { throw new ArgumentNullException("net"); } _net = net; _inputFormat = inputFormat; _encoder = new ReadonlyLockedValue <IVoiceEncoder>(encoder); //Create buffers to store the encoder input (1 frame of floats) and output (twice equivalent amount of bytes) _plainSamples = new float[encoder.FrameSize]; _encodedBytes = new byte[encoder.FrameSize * sizeof(float) * 2]; //Input buffer to store raw data from microphone _input = new BufferedSampleProvider(_inputFormat, encoder.FrameSize * 2); //Resample data from microphone rate -> encoder rate _resampler = new Resampler(_input, encoder.SampleRate); //Provides encoder sized and encoder rate frames of data _output = new SampleToFrameProvider(_resampler, (uint)encoder.FrameSize); }
public EncoderPipeline(IMicrophoneCapture mic, IVoiceEncoder encoder, ICommsNetwork net, Func <int> channelCount) { _mic = mic; _encoder = encoder; _net = net; _channelCount = channelCount; _encodedBytes = new byte[encoder.FrameSize * sizeof(float)]; _plainSamples = new float[encoder.FrameSize]; _inputFormat = mic.Format; //Create an input buffer with plenty of spare space _input = new BufferedSampleProvider(_inputFormat, Math.Max(_encoder.FrameSize * 2, mic.FrameSize * 2)); _resampler = new Resampler(_input, _encoder.SampleRate); //Whatever we did above, we need to read in frame size chunks _output = new SampleToFrameProvider(_resampler, (uint)encoder.FrameSize); }
/// <summary> /// /// </summary> /// <param name="inputFormat">format of audio supplied to `Send`</param> /// <param name="intermediateFrameSize">Size of frames which should be passed into `PreprocessAudioFrame`</param> /// <param name="intermediateSampleRate">Sample rate which should be passed into `PreprocessAudioFrame`</param> /// <param name="outputFrameSize">Size of frames which will be provided sent to `SendSamplesToSubscribers`</param> /// <param name="outputSampleRate"></param> protected BasePreprocessingPipeline([NotNull] WaveFormat inputFormat, int intermediateFrameSize, int intermediateSampleRate, int outputFrameSize, int outputSampleRate) { if (inputFormat == null) { throw new ArgumentNullException("inputFormat"); } if (intermediateFrameSize < 0) { throw new ArgumentOutOfRangeException("intermediateFrameSize", "Intermediate frame size cannot be less than zero"); } if (intermediateSampleRate < 0) { throw new ArgumentOutOfRangeException("intermediateSampleRate", "Intermediate sample rate cannot be less than zero"); } if (outputFrameSize < 0) { throw new ArgumentOutOfRangeException("outputFrameSize", "Output frame size cannot be less than zero"); } if (outputSampleRate < 0) { throw new ArgumentOutOfRangeException("outputSampleRate", "Output sample rate cannot be less than zero"); } _outputFrameSize = outputFrameSize; _outputFormat = new WaveFormat(outputSampleRate, 1); //Create resampler to resample input to intermediate rate _resamplerInput = new BufferedSampleProvider(inputFormat, intermediateFrameSize * 16); _resampler = new Resampler(_resamplerInput, 48000); _resampledOutput = new SampleToFrameProvider(_resampler, (uint)OutputFrameSize); _intermediateFrame = new float[intermediateFrameSize]; //Create a thread to drive the audio processing _threadEvent = new AutoResetEvent(false); _thread = new DThread(ThreadEntry); // We don't want to overestimate the latency. It's hard to come up with a reasonable number for this because if the input frames are >= the... // ...intermediate frames there will actually be no buffering delay in the preprocessor (it'll pick up a complete frame and process it right away). _estimatedPreprocessorLatencyMs = 0; }
/// <summary> /// /// </summary> /// <param name="inputFormat">format of audio supplied to `Send`</param> /// <param name="inputFrameSize">Size of frames which will should be provided from `GetFrameBuffer`</param> /// <param name="intermediateFrameSize">Size of frames which should be passed into `PreprocessAudioFrame`</param> /// <param name="intermediateSampleRate">Sample rate which should be passed into `PreprocessAudioFrame`</param> /// <param name="outputFrameSize">Size of frames which will be provided sent to `SendSamplesToSubscribers`</param> /// <param name="outputSampleRate"></param> protected BasePreprocessingPipeline([NotNull] WaveFormat inputFormat, int inputFrameSize, int intermediateFrameSize, int intermediateSampleRate, int outputFrameSize, int outputSampleRate) { if (inputFormat == null) { throw new ArgumentNullException("inputFormat"); } if (inputFrameSize < 0) { throw new ArgumentOutOfRangeException("inputFrameSize", "Input frame size cannot be less than zero"); } if (intermediateFrameSize < 0) { throw new ArgumentOutOfRangeException("intermediateFrameSize", "Intermediate frame size cannot be less than zero"); } if (intermediateSampleRate < 0) { throw new ArgumentOutOfRangeException("intermediateSampleRate", "Intermediate sample rate cannot be less than zero"); } if (outputFrameSize < 0) { throw new ArgumentOutOfRangeException("outputFrameSize", "Output frame size cannot be less than zero"); } if (outputSampleRate < 0) { throw new ArgumentOutOfRangeException("outputSampleRate", "Output sample rate cannot be less than zero"); } _inputFrameSize = inputFrameSize; _outputFrameSize = outputFrameSize; _outputFormat = new WaveFormat(1, outputSampleRate); //Create resampler to resample input to intermediate rate _resamplerInput = new BufferedSampleProvider(inputFormat, InputFrameSize * 16); _resampler = new Resampler(_resamplerInput, 48000); _resampledOutput = new SampleToFrameProvider(_resampler, (uint)OutputFrameSize); _intermediateFrame = new float[intermediateFrameSize]; _threadEvent = new AutoResetEvent(false); _thread = new DThread(ThreadEntry); }
/// <param name="micName"></param> /// <param name="source">Source to read frames from</param> private MicrophoneCapture([CanBeNull] string micName, AudioClip source) { if (source == null) { throw new ArgumentNullException("source", Log.PossibleBugMessage("capture source clip is null", "333E11A6-8026-41EB-9B34-EF9ADC54B651")); } _micName = micName; _clip = source; var captureFormat = new WaveFormat(1, source.frequency); _maxReadBufferPower = (byte)Math.Ceiling(Math.Log(0.1f * source.frequency, 2)); _preprocessing = new WebRtcPreprocessingPipeline(captureFormat); _preprocessing.Start(); //Ensure we have enough buffer size to contain several input frames to the preprocessor _rawMicSamples = new BufferedSampleProvider(captureFormat, _preprocessing.InputFrameSize * 4); _rawMicFrames = new SampleToFrameProvider(_rawMicSamples, (uint)_preprocessing.InputFrameSize); Log.Info("Began mic capture (SampleRate:{0} FrameSize:{1}, Buffer Limit:2^{2})", captureFormat.SampleRate, _preprocessing.InputFrameSize, _maxReadBufferPower); }
public virtual WaveFormat StartCapture(string inputMicName) { //Sanity checks Log.AssertAndThrowPossibleBug(_clip == null, "1BAD3E74-B451-4B7D-A9B9-35225BE55364", "Attempted to Start microphone capture, but capture is already running"); //Early exit if there are no microphones connected if (Log.AssertAndLogWarn(Microphone.devices.Length > 0, "No microphone detected; disabling voice capture")) { return(null); } //Check the micName and default to null if it's invalid (all whitespace or not a known device) _micName = ChooseMicName(inputMicName); //Get device capabilities and choose a sample rate as close to 48000 as possible. //If min and max are both zero that indicates we can use any sample rate int minFreq; int maxFreq; Microphone.GetDeviceCaps(_micName, out minFreq, out maxFreq); var sampleRate = minFreq == 0 && maxFreq == 0 ? 48000 : Mathf.Clamp(48000, minFreq, maxFreq); Log.Debug("GetDeviceCaps name=`{0}` min=`{1}` max=`{2}`", _micName, minFreq, maxFreq); //Get the audioclip from Unity for this microphone (with a fairly large internal buffer) _clip = Microphone.Start(_micName, true, 10, sampleRate); if (_clip == null) { Log.Error("Failed to start microphone capture"); return(null); } //Setup buffers for capture _format = new WaveFormat(_clip.frequency, 1); _maxReadBufferPower = (byte)Math.Ceiling(Math.Log(0.1f * _clip.frequency, 2)); // Create/resize the audio buffers to contain 20ms frames of data. Any frame size will work (the pipeline will buffer/split them as necessary) but 20ms is // optimal because that's native frame size the preprocessor works at so it has to do no extra work to assemble the frames at it's desired size. var frameSize = (int)(0.02 * _clip.frequency); if (_rawMicSamples == null || _rawMicSamples.WaveFormat != _format || _rawMicSamples.Capacity != frameSize || _rawMicFrames.FrameSize != frameSize) { _rawMicSamples = new BufferedSampleProvider(_format, frameSize * 4); _rawMicFrames = new SampleToFrameProvider(_rawMicSamples, (uint)frameSize); } if (_frame == null || _frame.Length != frameSize) { _frame = new float[frameSize]; } //watch for device changes - we need to reset if the audio device changes AudioSettings.OnAudioConfigurationChanged += OnAudioDeviceChanged; _audioDeviceChanged = false; //Reset subscribers to prepare them for another stream of data for (var i = 0; i < _subscribers.Count; i++) { _subscribers[i].Reset(); } Latency = TimeSpan.FromSeconds(frameSize / (float)_format.SampleRate); Log.Info("Began mic capture (SampleRate:{0}Hz, FrameSize:{1}, Buffer Limit:2^{2}, Latency:{3}ms, Device:'{4}')", _clip.frequency, frameSize, _maxReadBufferPower, Latency.TotalMilliseconds, _micName); return(_format); }