public void Synthesize(double[] f0, int f0Length, double[][] spectrogram, double[][] aperiodicity, int fftSize, double framePeriod, int fs, double[] y)
{
var minimumPhase = MinimumPhaseAnalysis.Create(fftSize);
var inverseRealFFT = InverseRealFFT.Create(fftSize);
var forwardRealFFT = ForwardRealFFT.Create(fftSize);
var pulseLocations = new double[y.Length];
var pulseLocationsIndex = new int[y.Length];
var pulseLocationsTimeShift = new double[y.Length];
var interpolatedVUV = new double[y.Length];
var numberOfPulses = GetTimeBase(f0, f0Length, fs, framePeriod / 1000.0, y.Length, fs / fftSize + 1.0, pulseLocations, pulseLocationsIndex, pulseLocationsTimeShift, interpolatedVUV);
var dcRemover = GetDCRemover(fftSize);
framePeriod /= 1000.0;
var impulseResponse = new double[fftSize];
for (var i = 0; i < numberOfPulses; i++)
{
var noiseSize = pulseLocationsIndex[Math.Min(numberOfPulses - 1, i + 1)] - pulseLocationsIndex[i];
GetOneFrameSegment(interpolatedVUV[pulseLocationsIndex[i]], noiseSize, spectrogram, fftSize, aperiodicity, f0Length, framePeriod, pulseLocations[i], pulseLocationsTimeShift[i], fs, forwardRealFFT, inverseRealFFT, minimumPhase, dcRemover, impulseResponse);
for (var j = 0; j < fftSize; j++)
{
var index = j + pulseLocationsIndex[i] - fftSize / 2 + 1;
if (index < 0 || index > y.Length - 1)
{
continue;
}
y[index] += impulseResponse[j];
}
}
}
//-----------------------------------------------------------------------------
// SmoothingWithRecovery() carries out the spectral smoothing and spectral
// recovery on the Cepstrum domain.
//-----------------------------------------------------------------------------
void SmoothingWithRecovery(double f0, int fs, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, double[] spectralEnvelope)
{
var smoothingLifter = new double[FFTSize];
var compensationLifter = new double[FFTSize];
smoothingLifter[0] = 1;
compensationLifter[0] = (1.0 - 2.0 * Q1) + 2.0 * Q1;
for (int i = 1, limit = forwardRealFFT.FFTSize / 2; i <= limit; i++)
{
var quefrency = (double)i / fs;
smoothingLifter[i] = Math.Sin(Math.PI * f0 * quefrency) / (Math.PI * f0 * quefrency);
compensationLifter[i] = (1.0 - 2.0 * Q1) + 2.0 * Q1 * Math.Cos(2.0 * Math.PI * quefrency * f0);
}
for (int i = 0, limit = FFTSize / 2; i <= limit; i++)
{
forwardRealFFT.Waveform[i] = Math.Log(forwardRealFFT.Waveform[i]);
}
for (int i = 1, limit = FFTSize / 2; i < limit; i++)
{
forwardRealFFT.Waveform[FFTSize - i] = forwardRealFFT.Waveform[i];
}
FFT.Execute(forwardRealFFT.ForwardFFT);
for (int i = 0, limit = FFTSize / 2; i <= limit; i++)
{
inverseRealFFT.Spectrum[i] = new Complex(forwardRealFFT.Spectrum[i].Real * smoothingLifter[i] * compensationLifter[i] / FFTSize, 0.0);
}
FFT.Execute(inverseRealFFT.InverseFFT);
for (int i = 0, limit = FFTSize / 2; i <= limit; i++)
{
spectralEnvelope[i] = Math.Exp(inverseRealFFT.Waveform[i]);
}
}
//-----------------------------------------------------------------------------
// GetSpectrumWithFractionalTimeShift() calculates a periodic spectrum with
// the fractional time shift under 1/fs.
//-----------------------------------------------------------------------------
void GetSpectrumWithFractionalTimeShift(int fftSize, double coefficient, InverseRealFFT inverseRealFFT)
{
for (var i = fftSize / 2; i > -1; i--)
{
var complex = inverseRealFFT.Spectrum[i];
var re2 = Math.Cos(coefficient * i);
var im2 = Math.Sqrt(1.0 - re2 * re2);
inverseRealFFT.Spectrum[i] = new Complex(complex.Real * re2 + complex.Imaginary * im2, complex.Imaginary * re2 - complex.Real * im2);
}
}
public SynthesisRealTime(int sampleRate, double framePeriod, int fftSize, int bufferSize, int ringBufferCapacity)
{
SampleRate = sampleRate;
FramePeriod = framePeriod * 0.001;
AudioBufferSize = bufferSize;
AudioBuffer = new double[bufferSize * 2 + fftSize];
Buffer = new RingBuffer(ringBufferCapacity);
FFTSize = fftSize;
ImpulseResponse = new double[fftSize];
DCRemover = GetDCRemover(fftSize / 2);
RefreshSynthesizer();
MinimumPhase = MinimumPhaseAnalysis.Create(fftSize);
InverseRealFFT = InverseRealFFT.Create(fftSize);
ForwardRealFFT = ForwardRealFFT.Create(fftSize);
}
public void Estimate(double[] x, int fs, double[] temporalPositions, double[] f0, double[][] spectrogram)
{
var f0Floor = GetF0FloorForCheapTrick(fs, FFTSize);
var spectralEnvelope = new double[FFTSize];
var forwardRealFFT = ForwardRealFFT.Create(FFTSize);
var inverseRealFFT = InverseRealFFT.Create(FFTSize);
for (var i = 0; i < f0.Length; i++)
{
var currentF0 = f0[i] <= f0Floor ? DefaultF0 : f0[i];
CheapTrickGeneralBody(x, fs, currentF0, temporalPositions[i], forwardRealFFT, inverseRealFFT, spectralEnvelope);
for (int j = 0, limit = FFTSize / 2; j <= limit; j++)
{
spectrogram[i][j] = spectralEnvelope[j];
}
}
}
static void FastFFTFilt(double[] x, double[] h, int fftSize, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, double[] y)
{
for (var i = 0; i < x.Length; i++)
{
forwardRealFFT.Waveform[i] = x[i] / fftSize;
}
if (x.Length < fftSize)
{
Array.Clear(forwardRealFFT.Waveform, x.Length, fftSize - x.Length);
}
FFT.Execute(forwardRealFFT.ForwardFFT);
var xSpectrum = new Complex[fftSize];
Array.Copy(forwardRealFFT.Spectrum, xSpectrum, fftSize / 2 + 1);
for (var i = 0; i < h.Length; i++)
{
forwardRealFFT.Waveform[i] = h[i] / fftSize;
}
if (h.Length < fftSize)
{
Array.Clear(forwardRealFFT.Waveform, h.Length, fftSize - h.Length);
}
FFT.Execute(forwardRealFFT.ForwardFFT);
for (var i = fftSize / 2 + 1; i > -1; i--)
{
inverseRealFFT.Spectrum[i] = new Complex(
xSpectrum[i].Real * forwardRealFFT.Spectrum[i].Real - xSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Imaginary,
xSpectrum[i].Real * forwardRealFFT.Spectrum[i].Imaginary + xSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Real
);
}
FFT.Execute(inverseRealFFT.InverseFFT);
inverseRealFFT.Waveform.BlockCopy(0, y, 0, fftSize);
}
//-----------------------------------------------------------------------------
// CheapTrickGeneralBody() calculates a spectral envelope at a temporal
// position. This function is only used in CheapTrick().
// Caution:
// forward_fft is allocated in advance to speed up the processing.
//-----------------------------------------------------------------------------
void CheapTrickGeneralBody(double[] x, int fs, double currentF0, double currentPosition, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, double[] spectralEnvelope)
{
// F0-adaptive windowing
GetWindowedWaveform(x, fs, currentF0, currentPosition, forwardRealFFT);
// Calculate power spectrum with DC correction
// Note: The calculated power spectrum is stored in an array for waveform.
// In this imprementation, power spectrum is transformed by FFT (NOT IFFT).
// However, the same result is obtained.
// This is tricky but important for simple implementation.
GetPowerSpectrum(fs, currentF0, forwardRealFFT);
// Smoothing of the power (linear axis)
// forward_real_fft.waveform is the power spectrum.
Common.LinearSmoothing(forwardRealFFT.Waveform, currentF0 * 2.0 / 3.0, fs, FFTSize, forwardRealFFT.Waveform);
// Add infinitesimal noise
// This is a safeguard to avoid including zero in the spectrum.
AddInfinitesimalNoise(forwardRealFFT.Waveform, FFTSize, forwardRealFFT.Waveform);
// Smoothing (log axis) and spectral recovery on the cepstrum domain.
SmoothingWithRecovery(currentF0, fs, forwardRealFFT, inverseRealFFT, spectralEnvelope);
}
//-----------------------------------------------------------------------------
// GetAperiodicResponse() calculates an aperiodic response.
//-----------------------------------------------------------------------------
void GetAperiodicResponse(int noiseSize, int fftSize, double[] spectrum, double[] aperiodicRatio, double currentVUV, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] aperiodicResponse)
{
GetNoiseSpectrum(noiseSize, fftSize, forwardRealFFT);
var logSpectrum = minimumPhase.LogSpectrum;
if (currentVUV != 0.0)
{
for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++)
{
logSpectrum[i] = Math.Log(spectrum[i] * aperiodicRatio[i]) / 2.0;
}
}
else
{
for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++)
{
logSpectrum[i] = Math.Log(spectrum[i]) / 2.0;
}
}
Common.GetMinimumPhaseSpectrum(minimumPhase);
var minimumPhaseSpectrum = minimumPhase.MinimumPhaseSpectrum;
Array.Copy(minimumPhaseSpectrum, 0, inverseRealFFT.Spectrum, 0, fftSize / 2 + 1);
for (int i = 0, limit = fftSize / 2; i <= limit; i++)
{
var real = minimumPhaseSpectrum[i].Real * forwardRealFFT.Spectrum[i].Real - minimumPhaseSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Imaginary;
var imaginary = minimumPhaseSpectrum[i].Real * forwardRealFFT.Spectrum[i].Imaginary + minimumPhaseSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Real;
inverseRealFFT.Spectrum[i] = new Complex(real, imaginary);
}
FFT.Execute(inverseRealFFT.InverseFFT);
MatlabFunctions.FFTShift(inverseRealFFT.Waveform.SubSequence(0, fftSize), aperiodicResponse);
}
//-----------------------------------------------------------------------------
// GetOneFrameSegment() calculates a periodic and aperiodic response at a time.
//-----------------------------------------------------------------------------
void GetOneFrameSegment(double currentVUV, int noiseSize, double[][] spectrogram, int fftSize, double[][] aperiodicity, int f0Length, double framePeriod, double currentTime, double fractionalTimeShift, int fs, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] dcRemover, double[] response)
{
var aperiodicResponse = new double[fftSize];
var periodicResponse = new double[fftSize];
var spectralEnvelope = new double[fftSize];
var aperiodicRatio = new double[fftSize];
GetSpectralEnvelope(currentTime, framePeriod, f0Length, spectrogram, fftSize, spectralEnvelope);
GetAperiodicRatio(currentTime, framePeriod, f0Length, aperiodicity, fftSize, aperiodicRatio);
// Synthesis of the periodic response
GetPeriodicResponse(fftSize, spectralEnvelope, aperiodicRatio, currentVUV, inverseRealFFT, minimumPhase, dcRemover, fractionalTimeShift, fs, periodicResponse);
// Synthesis of the aperiodic response
GetAperiodicResponse(noiseSize, fftSize, spectralEnvelope, aperiodicRatio, currentVUV, forwardRealFFT, inverseRealFFT, minimumPhase, aperiodicResponse);
var sqrtNoiseSize = Math.Sqrt(noiseSize);
for (var i = 0; i < fftSize; i++)
{
response[i] = (periodicResponse[i] * sqrtNoiseSize + aperiodicResponse[i]) / fftSize;
}
}
//-----------------------------------------------------------------------------
// GetPeriodicResponse() calculates an aperiodic response.
//-----------------------------------------------------------------------------
void GetPeriodicResponse(int fftSize, double[] spectrum, double[] aperiodicRatio, double currentVUV, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] dcRemover, double fractionalTimeShift, int fs, double[] periodicResponse)
{
if (currentVUV <= 0.5 || aperiodicRatio[0] > 0.999)
{
Array.Clear(periodicResponse, 0, fftSize);
return;
}
var logSpectrum = minimumPhase.LogSpectrum;
for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++)
{
logSpectrum[i] = Math.Log(spectrum[i] * (1.0 - aperiodicRatio[i]) + SafeGuardMinimum) / 2.0;
}
Common.GetMinimumPhaseSpectrum(minimumPhase);
Array.Copy(minimumPhase.MinimumPhaseSpectrum, 0, inverseRealFFT.Spectrum, 0, fftSize / 2 + 1);
double coefficient = PI2 * fractionalTimeShift * fs / fftSize;
GetSpectrumWithFractionalTimeShift(fftSize, coefficient, inverseRealFFT);
FFT.Execute(inverseRealFFT.InverseFFT);
MatlabFunctions.FFTShift(inverseRealFFT.Waveform.SubSequence(0, fftSize), periodicResponse);
RemoveDCComponent(periodicResponse, fftSize, dcRemover, periodicResponse);
}
