public void Synthesize(double[] f0, int f0Length, double[][] spectrogram, double[][] aperiodicity, int fftSize, double framePeriod, int fs, double[] y) { var minimumPhase = MinimumPhaseAnalysis.Create(fftSize); var inverseRealFFT = InverseRealFFT.Create(fftSize); var forwardRealFFT = ForwardRealFFT.Create(fftSize); var pulseLocations = new double[y.Length]; var pulseLocationsIndex = new int[y.Length]; var pulseLocationsTimeShift = new double[y.Length]; var interpolatedVUV = new double[y.Length]; var numberOfPulses = GetTimeBase(f0, f0Length, fs, framePeriod / 1000.0, y.Length, fs / fftSize + 1.0, pulseLocations, pulseLocationsIndex, pulseLocationsTimeShift, interpolatedVUV); var dcRemover = GetDCRemover(fftSize); framePeriod /= 1000.0; var impulseResponse = new double[fftSize]; for (var i = 0; i < numberOfPulses; i++) { var noiseSize = pulseLocationsIndex[Math.Min(numberOfPulses - 1, i + 1)] - pulseLocationsIndex[i]; GetOneFrameSegment(interpolatedVUV[pulseLocationsIndex[i]], noiseSize, spectrogram, fftSize, aperiodicity, f0Length, framePeriod, pulseLocations[i], pulseLocationsTimeShift[i], fs, forwardRealFFT, inverseRealFFT, minimumPhase, dcRemover, impulseResponse); for (var j = 0; j < fftSize; j++) { var index = j + pulseLocationsIndex[i] - fftSize / 2 + 1; if (index < 0 || index > y.Length - 1) { continue; } y[index] += impulseResponse[j]; } } }
public SynthesisRealTime(int sampleRate, double framePeriod, int fftSize, int bufferSize, int ringBufferCapacity) { SampleRate = sampleRate; FramePeriod = framePeriod * 0.001; AudioBufferSize = bufferSize; AudioBuffer = new double[bufferSize * 2 + fftSize]; Buffer = new RingBuffer(ringBufferCapacity); FFTSize = fftSize; ImpulseResponse = new double[fftSize]; DCRemover = GetDCRemover(fftSize / 2); RefreshSynthesizer(); MinimumPhase = MinimumPhaseAnalysis.Create(fftSize); InverseRealFFT = InverseRealFFT.Create(fftSize); ForwardRealFFT = ForwardRealFFT.Create(fftSize); }
internal static void GetMinimumPhaseSpectrum(MinimumPhaseAnalysis minimumPhase) { var fftSize = minimumPhase.FFTSize; for (var i = fftSize / 2 + 1; i < fftSize; i++) { minimumPhase.LogSpectrum[i] = minimumPhase.LogSpectrum[fftSize - i]; } FFT.Execute(minimumPhase.InverseFFT); var cepstrum = minimumPhase.Cepstrum; cepstrum[0] = new Complex(cepstrum[0].Real, cepstrum[0].Imaginary); for (int i = 1, limit = fftSize / 2; i < limit; i++) { cepstrum[i] = new Complex(cepstrum[i].Real * 2.0, cepstrum[i].Imaginary * -2.0); } cepstrum[fftSize / 2] = new Complex(cepstrum[fftSize / 2].Real, cepstrum[fftSize / 2].Imaginary * -1.0); for (var i = fftSize / 2 + 1; i < fftSize; i++) { cepstrum[i] = new Complex(); } FFT.Execute(minimumPhase.ForwardFFT); // Since x is complex number, calculation of exp(x) is as following. // Note: This FFT library does not keep the aliasing. var mininimumPhaseSpectrum = minimumPhase.MinimumPhaseSpectrum; for (int i = 0, limit = fftSize / 2; i <= limit; i++) { var tmp = Math.Exp(mininimumPhaseSpectrum[i].Real / fftSize); var real = tmp * Math.Cos(mininimumPhaseSpectrum[i].Imaginary / fftSize); var imaginary = tmp * Math.Sin(mininimumPhaseSpectrum[i].Imaginary / fftSize); mininimumPhaseSpectrum[i] = new Complex(real, imaginary); } }
//----------------------------------------------------------------------------- // GetAperiodicResponse() calculates an aperiodic response. //----------------------------------------------------------------------------- void GetAperiodicResponse(int noiseSize, int fftSize, double[] spectrum, double[] aperiodicRatio, double currentVUV, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] aperiodicResponse) { GetNoiseSpectrum(noiseSize, fftSize, forwardRealFFT); var logSpectrum = minimumPhase.LogSpectrum; if (currentVUV != 0.0) { for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++) { logSpectrum[i] = Math.Log(spectrum[i] * aperiodicRatio[i]) / 2.0; } } else { for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++) { logSpectrum[i] = Math.Log(spectrum[i]) / 2.0; } } Common.GetMinimumPhaseSpectrum(minimumPhase); var minimumPhaseSpectrum = minimumPhase.MinimumPhaseSpectrum; Array.Copy(minimumPhaseSpectrum, 0, inverseRealFFT.Spectrum, 0, fftSize / 2 + 1); for (int i = 0, limit = fftSize / 2; i <= limit; i++) { var real = minimumPhaseSpectrum[i].Real * forwardRealFFT.Spectrum[i].Real - minimumPhaseSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Imaginary; var imaginary = minimumPhaseSpectrum[i].Real * forwardRealFFT.Spectrum[i].Imaginary + minimumPhaseSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Real; inverseRealFFT.Spectrum[i] = new Complex(real, imaginary); } FFT.Execute(inverseRealFFT.InverseFFT); MatlabFunctions.FFTShift(inverseRealFFT.Waveform.SubSequence(0, fftSize), aperiodicResponse); }
//----------------------------------------------------------------------------- // GetOneFrameSegment() calculates a periodic and aperiodic response at a time. //----------------------------------------------------------------------------- void GetOneFrameSegment(double currentVUV, int noiseSize, double[][] spectrogram, int fftSize, double[][] aperiodicity, int f0Length, double framePeriod, double currentTime, double fractionalTimeShift, int fs, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] dcRemover, double[] response) { var aperiodicResponse = new double[fftSize]; var periodicResponse = new double[fftSize]; var spectralEnvelope = new double[fftSize]; var aperiodicRatio = new double[fftSize]; GetSpectralEnvelope(currentTime, framePeriod, f0Length, spectrogram, fftSize, spectralEnvelope); GetAperiodicRatio(currentTime, framePeriod, f0Length, aperiodicity, fftSize, aperiodicRatio); // Synthesis of the periodic response GetPeriodicResponse(fftSize, spectralEnvelope, aperiodicRatio, currentVUV, inverseRealFFT, minimumPhase, dcRemover, fractionalTimeShift, fs, periodicResponse); // Synthesis of the aperiodic response GetAperiodicResponse(noiseSize, fftSize, spectralEnvelope, aperiodicRatio, currentVUV, forwardRealFFT, inverseRealFFT, minimumPhase, aperiodicResponse); var sqrtNoiseSize = Math.Sqrt(noiseSize); for (var i = 0; i < fftSize; i++) { response[i] = (periodicResponse[i] * sqrtNoiseSize + aperiodicResponse[i]) / fftSize; } }
//----------------------------------------------------------------------------- // GetPeriodicResponse() calculates an aperiodic response. //----------------------------------------------------------------------------- void GetPeriodicResponse(int fftSize, double[] spectrum, double[] aperiodicRatio, double currentVUV, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] dcRemover, double fractionalTimeShift, int fs, double[] periodicResponse) { if (currentVUV <= 0.5 || aperiodicRatio[0] > 0.999) { Array.Clear(periodicResponse, 0, fftSize); return; } var logSpectrum = minimumPhase.LogSpectrum; for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++) { logSpectrum[i] = Math.Log(spectrum[i] * (1.0 - aperiodicRatio[i]) + SafeGuardMinimum) / 2.0; } Common.GetMinimumPhaseSpectrum(minimumPhase); Array.Copy(minimumPhase.MinimumPhaseSpectrum, 0, inverseRealFFT.Spectrum, 0, fftSize / 2 + 1); double coefficient = PI2 * fractionalTimeShift * fs / fftSize; GetSpectrumWithFractionalTimeShift(fftSize, coefficient, inverseRealFFT); FFT.Execute(inverseRealFFT.InverseFFT); MatlabFunctions.FFTShift(inverseRealFFT.Waveform.SubSequence(0, fftSize), periodicResponse); RemoveDCComponent(periodicResponse, fftSize, dcRemover, periodicResponse); }