/// <summary>
/// Creates a new Spectrogram object instance and performs a STFT on the given audio
/// </summary>
/// <param name="wav">a Wav object</param>
/// <param name="windowSize">is the size for the window in samples</param>
/// <param name="fps">is the desired frame rate</param>
/// <param name="online">work in online mode (i.e. use only past audio information)</param>
/// <param name="phase">include phase information</param>
public Spectrogram(Wav wav, MemoryAllocator allocator, int windowSize=2048, int fps=200, bool online=true, bool phase=true)
{
_allocator = allocator;
//init some variables
_wav = wav;
_fps = fps;
//derive some variables
HopSize = _wav.Samplerate / (float)_fps; //use floats so that seeking works properly
_frames = (int)(_wav.Samples / HopSize);
_ffts = windowSize / 2;
Bins = windowSize / 2; //initial number equal to ffts, can change if filters are used
//init STFT matrix
_STFT = _allocator.GetComplex32Matrix(_frames, _ffts);
//_STFT = DenseMatrix.Create(_frames, _ffts, Complex32.Zero);
//create windowing function
var cArray = wav.Audio.ToRowArrays()[0];
var values = MathNet.Numerics.Window.Hann(windowSize).Select(d => (float)d).ToArray();
Window = _allocator.GetFloatVector(values.Length);
Window.SetValues(values);
//Window = Vector<float>.Build.DenseOfArray(MathNet.Numerics.Window.Hann(windowSize).Select(d => (float)d).ToArray());
//step through all frames
System.Numerics.Complex[] result = new System.Numerics.Complex[Window.Count];
foreach (var frame in Enumerable.Range(0, _frames))
{
int seek;
Vector<float> signal;
//seek to the right position in the audio signal
if (online)
//step back a complete windowSize after moving forward 1 hopSize
//so that the current position is at the stop of the window
seek = (int)((frame + 1) * HopSize - windowSize);
else
//step back half of the windowSize so that the frame represents the centre of the window
seek = (int)(frame * HopSize - windowSize / 2);
//read in the right portion of the audio
if (seek >= _wav.Samples)
//stop of file reached
break;
else if (seek + windowSize > _wav.Samples)
{
//stop behind the actual audio stop, append zeros accordingly
int zeroAmount = seek + windowSize - _wav.Samples;
//var zeros = Vector<float>.Build.Dense(zeroAmount, 0);
var t = PythonUtilities.Slice<float>(cArray, seek, cArray.Length).ToArray();
//t.AddRange(zeros.ToList());
signal = _allocator.GetFloatVector(t.Length + zeroAmount);
for (int i = 0; i < t.Length; i++)
{
signal[i] = t[i];
}
//signal.SetValues(t);
//signal = Vector<float>.Build.DenseOfEnumerable(t);
}
else if (seek < 0)
{
//start before actual audio start, pad with zeros accordingly
int zeroAmount = -seek;
var zeros = Vector<float>.Build.Dense(zeroAmount, 0).ToList();
var t = PythonUtilities.Slice<float>(cArray, 0, seek + windowSize).ToArray();
zeros.AddRange(t);
signal = _allocator.GetFloatVector(t.Length + zeroAmount);
signal.SetValues(zeros.ToArray());
//signal = Vector<float>.Build.DenseOfEnumerable(zeros);
}
else
{
//normal read operation
var slice = PythonUtilities.Slice<float>(cArray, seek, seek + windowSize).ToArray();
signal = _allocator.GetFloatVector(slice.Length);
signal.SetValues(slice);
//signal = Vector<float>.Build.DenseOfEnumerable(PythonUtilities.Slice<float>(cArray, seek, seek + windowSize));
}
//multiply the signal with the window function
signal = signal.PointwiseMultiply(Window);
//only shift and perform complex DFT if needed
if (phase)
{
//circular shift the signal (needed for correct phase)
signal = NumpyCompatibility.FFTShift(signal);
}
//perform DFT
//sanity check
Debug.Assert(result.Length == signal.Count);
for (int i = 0; i < result.Length; i++)
{
result[i] = signal[i];
}
MathNet.Numerics.IntegralTransforms.Fourier.BluesteinForward(result, MathNet.Numerics.IntegralTransforms.FourierOptions.NoScaling);
_STFT.SetRow(frame, result.Select(r => new Complex32((float)r.Real, (float)r.Imaginary)).Take(_ffts).ToArray());
//var _newSTFTRow = result.Select(r => new Complex32((float)r.Real, (float)r.Imaginary)).Take(_ffts).ToArray();
//_STFT.SetRow(frame, _newSTFTRow);
//next frame
_allocator.ReturnFloatVectorStorage((MathNet.Numerics.LinearAlgebra.Storage.DenseVectorStorage<float>)signal.Storage);
}
//magnitude spectrogram
Spec = _allocator.GetFloatMatrix(_STFT.RowCount, _STFT.ColumnCount);
if (phase)
Phase = _allocator.GetFloatMatrix(_STFT.RowCount, _STFT.ColumnCount);
for (int i = 0; i < Spec.RowCount; i++)
{
for (int j = 0; j < Spec.ColumnCount; j++)
{
Spec.At(i, j, _STFT.At(i, j).Magnitude);
if (phase)
Phase.At(i, j, _STFT.At(i, j).Phase);
}
}
//Spec = _STFT.Map(c => (float)c.Magnitude);
//phase
//if (phase)
//{
// var imag = _STFT.Map(c => (float)c.Imaginary);
// var real = _STFT.Map(c => (float)c.Real);
// Phase = real.Map2((r, i) => (float)Math.Atan2(i,r), imag);
//}
}
public void LoadForFilter(string fileName, int filterLength, int windowSize, int cutoff, Func <int, int, double> window)
{
using (var reader = new WaveFileReader(fileName))
{
Samples = new List <float[]>();
var buffer = new byte[reader.Length];
var read = reader.Read(buffer, 0, buffer.Length);
var sampleBuffer = new short[read / 2];
Buffer.BlockCopy(buffer, 0, sampleBuffer, 0, read);
var samplingFrequency = reader.WaveFormat.SampleRate;
var filter = new double[filterLength];
var cutoffFrequency = cutoff;
for (int i = 0; i < filterLength; i++)
{
if (i == (filterLength - 1) / 2)
{
filter[i] = 2.0 * cutoffFrequency / samplingFrequency;
}
else
{
filter[i] = Math.Sin(((2 * Math.PI * cutoffFrequency) / samplingFrequency) *
(i - (filterLength - 1) / 2.0))
/ (Math.PI * (i - ((filterLength - 1) / 2.0)));
}
}
for (int i = 0; i < filterLength; i++)
{
filter[i] *= window(i, filterLength);
}
var globalFilterBuffer = new List <short>();
var windowLength = windowSize - filterLength + 1;
var windows = sampleBuffer.Length / windowLength;
var fourierN = windowLength + filterLength - 1;
var fourierFilter = filter.Select(d => new System.Numerics.Complex(d, 0)).ToArray();
fourierFilter = fourierFilter
.Concat(Enumerable.Repeat(new System.Numerics.Complex(0, 0), fourierN - filterLength)).ToArray();
FourierTransform.FFT(fourierFilter, FourierTransform.Direction.Forward);
for (int w = 0; w < windows; w++)
{
var samples = sampleBuffer.Skip(windowLength * w).Take(windowLength).ToArray();
samples =
samples
.Concat(Enumerable.Repeat((short)0, fourierN - windowLength)).ToArray();
var complexSamples = samples.Select(s => new System.Numerics.Complex(s, 0)).ToArray();
FourierTransform.FFT(complexSamples, FourierTransform.Direction.Forward);
var multiplied = new System.Numerics.Complex[complexSamples.Length];
for (int i = 0; i < complexSamples.Length; i++)
{
multiplied[i] = complexSamples[i] * fourierFilter[i];
}
FourierTransform.FFT(multiplied, FourierTransform.Direction.Backward);
globalFilterBuffer.AddRange(multiplied.Select(complex => (short)(complex.Real * 10)));
//var filtered = new short[windowLength + filterLength -1];
//for (int i = filterLength; i < samples.Length - filterLength; i++)
//{
// double y = 0;
// for (int j = 0; j < filterLength; j++)
// {
// y += samples[i - j] * filter[j];
// }
// filtered[i] = (short) y;
//}
}
Filtered = globalFilterBuffer.ToArray();
}
}
public void LoadForEqualizer(string fileName, List <FrequencyBand> bands)
{
using (var reader = new WaveFileReader(fileName))
{
Samples = new List <float[]>();
var buffer = new byte[reader.Length];
var read = reader.Read(buffer, 0, buffer.Length);
var sampleBuffer = new short[read / 2];
Buffer.BlockCopy(buffer, 0, sampleBuffer, 0, read);
var filterLength = 33;
var windowSize = 512;
var samplingFrequency = reader.WaveFormat.SampleRate;
var filters = new List <System.Numerics.Complex[]>();
var globalFilterBuffer = new List <short>();
var windowLength = windowSize - filterLength + 1;
var windows = sampleBuffer.Length / windowLength;
var fourierN = windowLength + filterLength - 1;
Points = new List <DataPoint>();
//for (int k = 0; k < 19980; k++)
//{
// Points.Add(new DataPoint(k, 0));
//}
//double[] x = Enumerable.Repeat(0.0, 1024).ToArray();
//x[0] = 1;// input signal (delta function example)
//var y = new List<double>(); // output signal
//ParametricEqualizer peq = new ParametricEqualizer(4, 1000, new double[] { 200, 250, 300, 350 }, new double[] { 5, 5, 5, 5 }, new double[] { 9, 9, 9, 9 }, new double[] { 0, 0, 0, 0 }, new double[] { 8, 10, 12, 14 });
//peq.run(x.ToList(), ref y);
//for (int i = 0; i < y.Count; i++)
//{
// Points.Add(new DataPoint(i, y[i]));
//}
double[] x = Enumerable.Repeat(0.0, samplingFrequency).ToArray();
x[0] = 1; // input signal (delta function example)
var y = new List <double>(); // output signal
var peq = new ParametricEqualizer(
bands.Count,
samplingFrequency,
bands.Select(band => (double)band.CenterFrequency).ToArray(),
bands.Select(band => (double)band.Width).ToArray(),
new double[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 },
new double[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
bands.Select(band => (double)band.Scale).ToArray());
peq.Run(x.ToList(), ref y);
for (int i = 0; i < y.Count; i++)
{
Points.Add(new DataPoint(i, y[i]));
}
//foreach (var band in bands.Take(1))
//{
//var filter = new double[filterLength];
//var f = new Section(band.CenterFrequency, band.Width/4.0, 9, 0, 3 * band.Scale, samplingFrequency);
//var test = new List<double>();
//for (int k = 20; k < 20000; k++)
//{
// test.Add(1);
//}
//f.run(test, out var outTest);
//for (int k = 0; k < 19980; k++)
//{
// Points[k] = new DataPoint(k, Points[k].Y + outTest[k]);
//}
//for (int i = 0; i < filterLength; i++)
//{
// //filter[i] = Math.Sqrt((1 + Math.Pow(i / band.StartFreq, 2)) / //boost above
// // (1 + Math.Pow(i / band.StopFreq, 2))) * band.Scale; //attuneate above
// //if ((i - (filterLength - 1) / 2) == 0)
// //{
// // filter[i] = ((i - (filterLength - 1) / 2.0) * (i - (filterLength - 1) / 2.0) +
// // band.CenterFrequency * band.CenterFrequency);
// //}
// //else
// //{
// // filter[i] = ((i - (filterLength - 1) / 2.0) * (i - (filterLength - 1) / 2.0) + band.CenterFrequency * band.CenterFrequency) / (i - (filterLength - 1) / 2.0) * (i - (filterLength - 1) / 2.0) + band.Width * (i - (filterLength - 1) / 2.0) +
// // band.CenterFrequency * band.CenterFrequency;
// //}
//}
//for (int i = 0; i < filterLength; i++)
//{
// filter[i] *= FastFourierTransform.HammingWindow(i, windowSize);
//}
//var fourierFilter = filter.Select(d => new System.Numerics.Complex(d, 0)).ToArray();
//fourierFilter = fourierFilter
// .Concat(Enumerable.Repeat(new System.Numerics.Complex(0, 0), fourierN - filterLength)).ToArray();
//FourierTransform.FFT(fourierFilter, FourierTransform.Direction.Forward);
//filters.Add(fourierFilter);
//}
var fourierFilter = y.Take(fourierN).ToArray();
var complexPoints = Points.Select(point => new System.Numerics.Complex(point.Y, 0)).ToArray();
FourierTransform2.FFT(complexPoints, FourierTransform.Direction.Forward);
int z = 0;
Points = complexPoints.Take(complexPoints.Length / 2).Select(complex => new DataPoint(z++ *1000 / 1024.0, complex.Magnitude)).ToList();
for (int w = 0; w < windows; w++)
{
var samples = sampleBuffer.Skip(windowLength * w).Take(windowLength).ToArray();
samples =
samples
.Concat(Enumerable.Repeat((short)0, fourierN - windowLength)).ToArray();
var complexSamples = samples.Select(s => new System.Numerics.Complex(s, 0)).ToArray();
FourierTransform.FFT(complexSamples, FourierTransform.Direction.Forward);
var multiplied = new System.Numerics.Complex[complexSamples.Length];
for (int i = 0; i < complexSamples.Length; i++)
{
multiplied[i] = complexSamples[i] * fourierFilter[i];
}
FourierTransform.FFT(multiplied, FourierTransform.Direction.Backward);
globalFilterBuffer.AddRange(multiplied.Select(complex => (short)(complex.Real * 10)));
//var filtered = new short[windowLength + filterLength -1];
//for (int i = filterLength; i < samples.Length - filterLength; i++)
//{
// double y = 0;
// for (int j = 0; j < filterLength; j++)
// {
// y += samples[i - j] * filter[j];
// }
// filtered[i] = (short) y;
//}
}
Filtered = globalFilterBuffer.ToArray();
}
}
/// <summary>
/// Creates a new Spectrogram object instance and performs a STFT on the given audio
/// </summary>
/// <param name="wav">a Wav object</param>
/// <param name="windowSize">is the size for the window in samples</param>
/// <param name="fps">is the desired frame rate</param>
/// <param name="online">work in online mode (i.e. use only past audio information)</param>
/// <param name="phase">include phase information</param>
public Spectrogram(Wav wav, MemoryAllocator allocator, int windowSize = 2048, int fps = 200, bool online = true, bool phase = true)
{
_allocator = allocator;
//init some variables
_wav = wav;
_fps = fps;
//derive some variables
HopSize = _wav.Samplerate / (float)_fps; //use floats so that seeking works properly
_frames = (int)(_wav.Samples / HopSize);
_ffts = windowSize / 2;
Bins = windowSize / 2; //initial number equal to ffts, can change if filters are used
//init STFT matrix
_STFT = _allocator.GetComplex32Matrix(_frames, _ffts);
//_STFT = DenseMatrix.Create(_frames, _ffts, Complex32.Zero);
//create windowing function
var cArray = wav.Audio.ToRowArrays()[0];
var values = MathNet.Numerics.Window.Hann(windowSize).Select(d => (float)d).ToArray();
Window = _allocator.GetFloatVector(values.Length);
Window.SetValues(values);
//Window = Vector<float>.Build.DenseOfArray(MathNet.Numerics.Window.Hann(windowSize).Select(d => (float)d).ToArray());
//step through all frames
System.Numerics.Complex[] result = new System.Numerics.Complex[Window.Count];
foreach (var frame in Enumerable.Range(0, _frames))
{
int seek;
Vector <float> signal;
//seek to the right position in the audio signal
if (online)
{
//step back a complete windowSize after moving forward 1 hopSize
//so that the current position is at the stop of the window
seek = (int)((frame + 1) * HopSize - windowSize);
}
else
{
//step back half of the windowSize so that the frame represents the centre of the window
seek = (int)(frame * HopSize - windowSize / 2);
}
//read in the right portion of the audio
if (seek >= _wav.Samples)
{
//stop of file reached
break;
}
else if (seek + windowSize > _wav.Samples)
{
//stop behind the actual audio stop, append zeros accordingly
int zeroAmount = seek + windowSize - _wav.Samples;
//var zeros = Vector<float>.Build.Dense(zeroAmount, 0);
var t = PythonUtilities.Slice <float>(cArray, seek, cArray.Length).ToArray();
//t.AddRange(zeros.ToList());
signal = _allocator.GetFloatVector(t.Length + zeroAmount);
for (int i = 0; i < t.Length; i++)
{
signal[i] = t[i];
}
//signal.SetValues(t);
//signal = Vector<float>.Build.DenseOfEnumerable(t);
}
else if (seek < 0)
{
//start before actual audio start, pad with zeros accordingly
int zeroAmount = -seek;
var zeros = Vector <float> .Build.Dense(zeroAmount, 0).ToList();
var t = PythonUtilities.Slice <float>(cArray, 0, seek + windowSize).ToArray();
zeros.AddRange(t);
signal = _allocator.GetFloatVector(t.Length + zeroAmount);
signal.SetValues(zeros.ToArray());
//signal = Vector<float>.Build.DenseOfEnumerable(zeros);
}
else
{
//normal read operation
var slice = PythonUtilities.Slice <float>(cArray, seek, seek + windowSize).ToArray();
signal = _allocator.GetFloatVector(slice.Length);
signal.SetValues(slice);
//signal = Vector<float>.Build.DenseOfEnumerable(PythonUtilities.Slice<float>(cArray, seek, seek + windowSize));
}
//multiply the signal with the window function
signal = signal.PointwiseMultiply(Window);
//only shift and perform complex DFT if needed
if (phase)
{
//circular shift the signal (needed for correct phase)
signal = NumpyCompatibility.FFTShift(signal);
}
//perform DFT
//sanity check
Debug.Assert(result.Length == signal.Count);
for (int i = 0; i < result.Length; i++)
{
result[i] = signal[i];
}
MathNet.Numerics.IntegralTransforms.Fourier.BluesteinForward(result, MathNet.Numerics.IntegralTransforms.FourierOptions.NoScaling);
_STFT.SetRow(frame, result.Select(r => new Complex32((float)r.Real, (float)r.Imaginary)).Take(_ffts).ToArray());
//var _newSTFTRow = result.Select(r => new Complex32((float)r.Real, (float)r.Imaginary)).Take(_ffts).ToArray();
//_STFT.SetRow(frame, _newSTFTRow);
//next frame
_allocator.ReturnFloatVectorStorage((MathNet.Numerics.LinearAlgebra.Storage.DenseVectorStorage <float>)signal.Storage);
}
//magnitude spectrogram
Spec = _allocator.GetFloatMatrix(_STFT.RowCount, _STFT.ColumnCount);
if (phase)
{
Phase = _allocator.GetFloatMatrix(_STFT.RowCount, _STFT.ColumnCount);
}
for (int i = 0; i < Spec.RowCount; i++)
{
for (int j = 0; j < Spec.ColumnCount; j++)
{
Spec.At(i, j, _STFT.At(i, j).Magnitude);
if (phase)
{
Phase.At(i, j, _STFT.At(i, j).Phase);
}
}
}
//Spec = _STFT.Map(c => (float)c.Magnitude);
//phase
//if (phase)
//{
// var imag = _STFT.Map(c => (float)c.Imaginary);
// var real = _STFT.Map(c => (float)c.Real);
// Phase = real.Map2((r, i) => (float)Math.Atan2(i,r), imag);
//}
}
