public void Synthesize(double[] f0, int f0Length, double[][] spectrogram, double[][] aperiodicity, int fftSize, double framePeriod, int fs, double[] y)
{
var minimumPhase = MinimumPhaseAnalysis.Create(fftSize);
var inverseRealFFT = InverseRealFFT.Create(fftSize);
var forwardRealFFT = ForwardRealFFT.Create(fftSize);
var pulseLocations = new double[y.Length];
var pulseLocationsIndex = new int[y.Length];
var pulseLocationsTimeShift = new double[y.Length];
var interpolatedVUV = new double[y.Length];
var numberOfPulses = GetTimeBase(f0, f0Length, fs, framePeriod / 1000.0, y.Length, fs / fftSize + 1.0, pulseLocations, pulseLocationsIndex, pulseLocationsTimeShift, interpolatedVUV);
var dcRemover = GetDCRemover(fftSize);
framePeriod /= 1000.0;
var impulseResponse = new double[fftSize];
for (var i = 0; i < numberOfPulses; i++)
{
var noiseSize = pulseLocationsIndex[Math.Min(numberOfPulses - 1, i + 1)] - pulseLocationsIndex[i];
GetOneFrameSegment(interpolatedVUV[pulseLocationsIndex[i]], noiseSize, spectrogram, fftSize, aperiodicity, f0Length, framePeriod, pulseLocations[i], pulseLocationsTimeShift[i], fs, forwardRealFFT, inverseRealFFT, minimumPhase, dcRemover, impulseResponse);
for (var j = 0; j < fftSize; j++)
{
var index = j + pulseLocationsIndex[i] - fftSize / 2 + 1;
if (index < 0 || index > y.Length - 1)
{
continue;
}
y[index] += impulseResponse[j];
}
}
}
public SynthesisRealTime(int sampleRate, double framePeriod, int fftSize, int bufferSize, int ringBufferCapacity)
{
SampleRate = sampleRate;
FramePeriod = framePeriod * 0.001;
AudioBufferSize = bufferSize;
AudioBuffer = new double[bufferSize * 2 + fftSize];
Buffer = new RingBuffer(ringBufferCapacity);
FFTSize = fftSize;
ImpulseResponse = new double[fftSize];
DCRemover = GetDCRemover(fftSize / 2);
RefreshSynthesizer();
MinimumPhase = MinimumPhaseAnalysis.Create(fftSize);
InverseRealFFT = InverseRealFFT.Create(fftSize);
ForwardRealFFT = ForwardRealFFT.Create(fftSize);
}
internal static void GetMinimumPhaseSpectrum(MinimumPhaseAnalysis minimumPhase)
{
var fftSize = minimumPhase.FFTSize;
for (var i = fftSize / 2 + 1; i < fftSize; i++)
{
minimumPhase.LogSpectrum[i] = minimumPhase.LogSpectrum[fftSize - i];
}
FFT.Execute(minimumPhase.InverseFFT);
var cepstrum = minimumPhase.Cepstrum;
cepstrum[0] = new Complex(cepstrum[0].Real, cepstrum[0].Imaginary);
for (int i = 1, limit = fftSize / 2; i < limit; i++)
{
cepstrum[i] = new Complex(cepstrum[i].Real * 2.0, cepstrum[i].Imaginary * -2.0);
}
cepstrum[fftSize / 2] = new Complex(cepstrum[fftSize / 2].Real, cepstrum[fftSize / 2].Imaginary * -1.0);
for (var i = fftSize / 2 + 1; i < fftSize; i++)
{
cepstrum[i] = new Complex();
}
FFT.Execute(minimumPhase.ForwardFFT);
// Since x is complex number, calculation of exp(x) is as following.
// Note: This FFT library does not keep the aliasing.
var mininimumPhaseSpectrum = minimumPhase.MinimumPhaseSpectrum;
for (int i = 0, limit = fftSize / 2; i <= limit; i++)
{
var tmp = Math.Exp(mininimumPhaseSpectrum[i].Real / fftSize);
var real = tmp * Math.Cos(mininimumPhaseSpectrum[i].Imaginary / fftSize);
var imaginary = tmp * Math.Sin(mininimumPhaseSpectrum[i].Imaginary / fftSize);
mininimumPhaseSpectrum[i] = new Complex(real, imaginary);
}
}
//-----------------------------------------------------------------------------
// GetAperiodicResponse() calculates an aperiodic response.
//-----------------------------------------------------------------------------
void GetAperiodicResponse(int noiseSize, int fftSize, double[] spectrum, double[] aperiodicRatio, double currentVUV, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] aperiodicResponse)
{
GetNoiseSpectrum(noiseSize, fftSize, forwardRealFFT);
var logSpectrum = minimumPhase.LogSpectrum;
if (currentVUV != 0.0)
{
for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++)
{
logSpectrum[i] = Math.Log(spectrum[i] * aperiodicRatio[i]) / 2.0;
}
}
else
{
for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++)
{
logSpectrum[i] = Math.Log(spectrum[i]) / 2.0;
}
}
Common.GetMinimumPhaseSpectrum(minimumPhase);
var minimumPhaseSpectrum = minimumPhase.MinimumPhaseSpectrum;
Array.Copy(minimumPhaseSpectrum, 0, inverseRealFFT.Spectrum, 0, fftSize / 2 + 1);
for (int i = 0, limit = fftSize / 2; i <= limit; i++)
{
var real = minimumPhaseSpectrum[i].Real * forwardRealFFT.Spectrum[i].Real - minimumPhaseSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Imaginary;
var imaginary = minimumPhaseSpectrum[i].Real * forwardRealFFT.Spectrum[i].Imaginary + minimumPhaseSpectrum[i].Imaginary * forwardRealFFT.Spectrum[i].Real;
inverseRealFFT.Spectrum[i] = new Complex(real, imaginary);
}
FFT.Execute(inverseRealFFT.InverseFFT);
MatlabFunctions.FFTShift(inverseRealFFT.Waveform.SubSequence(0, fftSize), aperiodicResponse);
}
//-----------------------------------------------------------------------------
// GetOneFrameSegment() calculates a periodic and aperiodic response at a time.
//-----------------------------------------------------------------------------
void GetOneFrameSegment(double currentVUV, int noiseSize, double[][] spectrogram, int fftSize, double[][] aperiodicity, int f0Length, double framePeriod, double currentTime, double fractionalTimeShift, int fs, ForwardRealFFT forwardRealFFT, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] dcRemover, double[] response)
{
var aperiodicResponse = new double[fftSize];
var periodicResponse = new double[fftSize];
var spectralEnvelope = new double[fftSize];
var aperiodicRatio = new double[fftSize];
GetSpectralEnvelope(currentTime, framePeriod, f0Length, spectrogram, fftSize, spectralEnvelope);
GetAperiodicRatio(currentTime, framePeriod, f0Length, aperiodicity, fftSize, aperiodicRatio);
// Synthesis of the periodic response
GetPeriodicResponse(fftSize, spectralEnvelope, aperiodicRatio, currentVUV, inverseRealFFT, minimumPhase, dcRemover, fractionalTimeShift, fs, periodicResponse);
// Synthesis of the aperiodic response
GetAperiodicResponse(noiseSize, fftSize, spectralEnvelope, aperiodicRatio, currentVUV, forwardRealFFT, inverseRealFFT, minimumPhase, aperiodicResponse);
var sqrtNoiseSize = Math.Sqrt(noiseSize);
for (var i = 0; i < fftSize; i++)
{
response[i] = (periodicResponse[i] * sqrtNoiseSize + aperiodicResponse[i]) / fftSize;
}
}
//-----------------------------------------------------------------------------
// GetPeriodicResponse() calculates an aperiodic response.
//-----------------------------------------------------------------------------
void GetPeriodicResponse(int fftSize, double[] spectrum, double[] aperiodicRatio, double currentVUV, InverseRealFFT inverseRealFFT, MinimumPhaseAnalysis minimumPhase, double[] dcRemover, double fractionalTimeShift, int fs, double[] periodicResponse)
{
if (currentVUV <= 0.5 || aperiodicRatio[0] > 0.999)
{
Array.Clear(periodicResponse, 0, fftSize);
return;
}
var logSpectrum = minimumPhase.LogSpectrum;
for (int i = 0, limit = minimumPhase.FFTSize / 2; i <= limit; i++)
{
logSpectrum[i] = Math.Log(spectrum[i] * (1.0 - aperiodicRatio[i]) + SafeGuardMinimum) / 2.0;
}
Common.GetMinimumPhaseSpectrum(minimumPhase);
Array.Copy(minimumPhase.MinimumPhaseSpectrum, 0, inverseRealFFT.Spectrum, 0, fftSize / 2 + 1);
double coefficient = PI2 * fractionalTimeShift * fs / fftSize;
GetSpectrumWithFractionalTimeShift(fftSize, coefficient, inverseRealFFT);
FFT.Execute(inverseRealFFT.InverseFFT);
MatlabFunctions.FFTShift(inverseRealFFT.Waveform.SubSequence(0, fftSize), periodicResponse);
RemoveDCComponent(periodicResponse, fftSize, dcRemover, periodicResponse);
}
