/// <summary>
/// Creates a new Spectrogram object instance and performs a STFT on the given audio
/// </summary>
/// <param name="wav">a Wav object</param>
/// <param name="windowSize">is the size for the window in samples</param>
/// <param name="fps">is the desired frame rate</param>
/// <param name="online">work in online mode (i.e. use only past audio information)</param>
/// <param name="phase">include phase information</param>
public Spectrogram(Wav wav, MemoryAllocator allocator, int windowSize = 2048, int fps = 200, bool online = true, bool phase = true)
{
_allocator = allocator;
//init some variables
_wav = wav;
_fps = fps;
//derive some variables
HopSize = _wav.Samplerate / (float)_fps; //use floats so that seeking works properly
_frames = (int)(_wav.Samples / HopSize);
_ffts = windowSize / 2;
Bins = windowSize / 2; //initial number equal to ffts, can change if filters are used
//init STFT matrix
_STFT = _allocator.GetComplex32Matrix(_frames, _ffts);
//_STFT = DenseMatrix.Create(_frames, _ffts, Complex32.Zero);
//create windowing function
var cArray = wav.Audio.ToRowArrays()[0];
var values = MathNet.Numerics.Window.Hann(windowSize).Select(d => (float)d).ToArray();
Window = _allocator.GetFloatVector(values.Length);
Window.SetValues(values);
//Window = Vector<float>.Build.DenseOfArray(MathNet.Numerics.Window.Hann(windowSize).Select(d => (float)d).ToArray());
//step through all frames
System.Numerics.Complex[] result = new System.Numerics.Complex[Window.Count];
foreach (var frame in Enumerable.Range(0, _frames))
{
int seek;
Vector <float> signal;
//seek to the right position in the audio signal
if (online)
{
//step back a complete windowSize after moving forward 1 hopSize
//so that the current position is at the stop of the window
seek = (int)((frame + 1) * HopSize - windowSize);
}
else
{
//step back half of the windowSize so that the frame represents the centre of the window
seek = (int)(frame * HopSize - windowSize / 2);
}
//read in the right portion of the audio
if (seek >= _wav.Samples)
{
//stop of file reached
break;
}
else if (seek + windowSize > _wav.Samples)
{
//stop behind the actual audio stop, append zeros accordingly
int zeroAmount = seek + windowSize - _wav.Samples;
//var zeros = Vector<float>.Build.Dense(zeroAmount, 0);
var t = PythonUtilities.Slice <float>(cArray, seek, cArray.Length).ToArray();
//t.AddRange(zeros.ToList());
signal = _allocator.GetFloatVector(t.Length + zeroAmount);
for (int i = 0; i < t.Length; i++)
{
signal[i] = t[i];
}
//signal.SetValues(t);
//signal = Vector<float>.Build.DenseOfEnumerable(t);
}
else if (seek < 0)
{
//start before actual audio start, pad with zeros accordingly
int zeroAmount = -seek;
var zeros = Vector <float> .Build.Dense(zeroAmount, 0).ToList();
var t = PythonUtilities.Slice <float>(cArray, 0, seek + windowSize).ToArray();
zeros.AddRange(t);
signal = _allocator.GetFloatVector(t.Length + zeroAmount);
signal.SetValues(zeros.ToArray());
//signal = Vector<float>.Build.DenseOfEnumerable(zeros);
}
else
{
//normal read operation
var slice = PythonUtilities.Slice <float>(cArray, seek, seek + windowSize).ToArray();
signal = _allocator.GetFloatVector(slice.Length);
signal.SetValues(slice);
//signal = Vector<float>.Build.DenseOfEnumerable(PythonUtilities.Slice<float>(cArray, seek, seek + windowSize));
}
//multiply the signal with the window function
signal = signal.PointwiseMultiply(Window);
//only shift and perform complex DFT if needed
if (phase)
{
//circular shift the signal (needed for correct phase)
signal = NumpyCompatibility.FFTShift(signal);
}
//perform DFT
//sanity check
Debug.Assert(result.Length == signal.Count);
for (int i = 0; i < result.Length; i++)
{
result[i] = signal[i];
}
MathNet.Numerics.IntegralTransforms.Fourier.BluesteinForward(result, MathNet.Numerics.IntegralTransforms.FourierOptions.NoScaling);
_STFT.SetRow(frame, result.Select(r => new Complex32((float)r.Real, (float)r.Imaginary)).Take(_ffts).ToArray());
//var _newSTFTRow = result.Select(r => new Complex32((float)r.Real, (float)r.Imaginary)).Take(_ffts).ToArray();
//_STFT.SetRow(frame, _newSTFTRow);
//next frame
_allocator.ReturnFloatVectorStorage((MathNet.Numerics.LinearAlgebra.Storage.DenseVectorStorage <float>)signal.Storage);
}
//magnitude spectrogram
Spec = _allocator.GetFloatMatrix(_STFT.RowCount, _STFT.ColumnCount);
if (phase)
{
Phase = _allocator.GetFloatMatrix(_STFT.RowCount, _STFT.ColumnCount);
}
for (int i = 0; i < Spec.RowCount; i++)
{
for (int j = 0; j < Spec.ColumnCount; j++)
{
Spec.At(i, j, _STFT.At(i, j).Magnitude);
if (phase)
{
Phase.At(i, j, _STFT.At(i, j).Phase);
}
}
}
//Spec = _STFT.Map(c => (float)c.Magnitude);
//phase
//if (phase)
//{
// var imag = _STFT.Map(c => (float)c.Imaginary);
// var real = _STFT.Map(c => (float)c.Real);
// Phase = real.Map2((r, i) => (float)Math.Atan2(i,r), imag);
//}
}
/// <summary>
/// Calculates a triangular window of the given size.
/// </summary>
/// <param name="start">starting bin (with value 0, included in the returned filter)</param>
/// <param name="mid">center bin (of height 1, unless norm is True)</param>
/// <param name="stop">end bin (with value 0, not included in the returned filter)</param>
/// <param name="equal">normalize the area of the filter to 1 [default=False]</param>
/// <returns>a triangular shaped filter</returns>
public static float[] Triang(int start, int mid, int stop, bool equal = false)
{
//height of the filter
var height = 1.0f;
//normalize the height
if (equal)
{
height = 2f / (stop - start);
}
//init the filter
float[] triang_filter = new float[stop - start];
//rising edge
PythonUtilities.SetArraySegment(PythonUtilities.Slice(triang_filter, 0, mid - start), NumpyCompatibility.LinSpace(0, height, (mid - start), false).ToArray());
//falling edge
PythonUtilities.SetArraySegment(PythonUtilities.Slice(triang_filter, mid - start, triang_filter.Length), NumpyCompatibility.LinSpace(height, 0, (stop - mid), false).ToArray());
//return
return(triang_filter);
}
