/// <summary>
/// Constructor
/// </summary>
/// <param name="samplingRate"></param>
/// <param name="featureList"></param>
/// <param name="frameDuration"></param>
/// <param name="hopDuration"></param>
/// <param name="fftSize"></param>
/// <param name="frequencyBands"></param>
/// <param name="parameters"></param>
public Mpeg7SpectralFeaturesExtractor(int samplingRate,
string featureList,
double frameDuration = 0.0256 /*sec*/,
double hopDuration = 0.010 /*sec*/,
int fftSize = 0,
Tuple <double, double, double>[] frequencyBands = null,
WindowTypes window = WindowTypes.Hamming,
IReadOnlyDictionary <string, object> parameters = null)
: base(samplingRate, frameDuration, hopDuration)
{
if (featureList == "all" || featureList == "full")
{
featureList = FeatureSet;
}
var features = featureList.Split(',', '+', '-', ';', ':')
.Select(f => f.Trim().ToLower());
_extractors = features.Select <string, Func <float[], float[], float> >(feature =>
{
switch (feature)
{
case "sc":
case "centroid":
return(Spectral.Centroid);
case "ss":
case "spread":
return(Spectral.Spread);
case "sfm":
case "flatness":
if (parameters?.ContainsKey("minLevel") ?? false)
{
var minLevel = (float)parameters["minLevel"];
return((spectrum, freqs) => Spectral.Flatness(spectrum, minLevel));
}
else
{
return((spectrum, freqs) => Spectral.Flatness(spectrum));
}
case "sn":
case "noiseness":
if (parameters?.ContainsKey("noiseFrequency") ?? false)
{
var noiseFrequency = (float)parameters["noiseFrequency"];
return((spectrum, freqs) => Spectral.Noiseness(spectrum, freqs, noiseFrequency));
}
else
{
return((spectrum, freqs) => Spectral.Noiseness(spectrum, freqs));
}
case "rolloff":
if (parameters?.ContainsKey("rolloffPercent") ?? false)
{
var rolloffPercent = (float)parameters["rolloffPercent"];
return((spectrum, freqs) => Spectral.Rolloff(spectrum, freqs, rolloffPercent));
}
else
{
return((spectrum, freqs) => Spectral.Rolloff(spectrum, freqs));
}
case "crest":
return((spectrum, freqs) => Spectral.Crest(spectrum));
case "entropy":
case "ent":
return((spectrum, freqs) => Spectral.Entropy(spectrum));
case "sd":
case "decrease":
return((spectrum, freqs) => Spectral.Decrease(spectrum));
case "loud":
case "loudness":
return((spectrum, freqs) => Perceptual.Loudness(spectrum));
case "sharp":
case "sharpness":
return((spectrum, freqs) => Perceptual.Sharpness(spectrum));
default:
return((spectrum, freqs) => 0);
}
}).ToList();
FeatureDescriptions = features.ToList();
_fftSize = fftSize > FrameSize ? fftSize : MathUtils.NextPowerOfTwo(FrameSize);
_fft = new Fft(_fftSize);
_window = window;
if (_window != WindowTypes.Rectangular)
{
_windowSamples = Window.OfType(_window, FrameSize);
}
_frequencyBands = frequencyBands ?? FilterBanks.OctaveBands(6, _fftSize, samplingRate);
_filterbank = FilterBanks.Rectangular(_fftSize, samplingRate, _frequencyBands);
var cfs = _frequencyBands.Select(b => b.Item2).ToList();
// insert zero frequency so that it'll be ignored during calculations
// just like in case of FFT spectrum (0th DC component)
cfs.Insert(0, 0);
_frequencies = cfs.ToFloats();
_parameters = parameters;
// reserve memory for reusable blocks
_spectrum = new float[_fftSize / 2 + 1]; // buffer for magnitude spectrum
_mappedSpectrum = new float[_filterbank.Length + 1]; // buffer for total energies in bands
_block = new float[_fftSize]; // buffer for currently processed block
_zeroblock = new float[_fftSize]; // just a buffer of zeros for quick memset
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="options">Filterbank options</param>
public FilterbankExtractor(FilterbankOptions options) : base(options)
{
var filterbankSize = options.FilterBankSize;
if (options.FilterBank == null)
{
_blockSize = options.FftSize > FrameSize ? options.FftSize : MathUtils.NextPowerOfTwo(FrameSize);
var melBands = FilterBanks.MelBands(filterbankSize, SamplingRate, options.LowFrequency, options.HighFrequency, false);
FilterBank = FilterBanks.Rectangular(_blockSize, SamplingRate, melBands, mapper: Scale.HerzToMel);
}
else
{
FilterBank = options.FilterBank;
filterbankSize = FilterBank.Length;
_blockSize = 2 * (FilterBank[0].Length - 1);
Guard.AgainstExceedance(FrameSize, _blockSize, "frame size", "FFT size");
}
FeatureCount = filterbankSize;
_fft = new RealFft(_blockSize);
// setup spectrum post-processing: =======================================================
_logFloor = options.LogFloor;
_nonLinearityType = options.NonLinearity;
switch (_nonLinearityType)
{
case NonLinearityType.Log10:
_postProcessSpectrum = () => FilterBanks.ApplyAndLog10(FilterBank, _spectrum, _bandSpectrum, _logFloor); break;
case NonLinearityType.LogE:
_postProcessSpectrum = () => FilterBanks.ApplyAndLog(FilterBank, _spectrum, _bandSpectrum, _logFloor); break;
case NonLinearityType.ToDecibel:
_postProcessSpectrum = () => FilterBanks.ApplyAndToDecibel(FilterBank, _spectrum, _bandSpectrum, _logFloor); break;
case NonLinearityType.CubicRoot:
_postProcessSpectrum = () => FilterBanks.ApplyAndPow(FilterBank, _spectrum, _bandSpectrum, 0.33); break;
default:
_postProcessSpectrum = () => FilterBanks.Apply(FilterBank, _spectrum, _bandSpectrum); break;
}
_spectrumType = options.SpectrumType;
switch (_spectrumType)
{
case SpectrumType.Magnitude:
_getSpectrum = block => _fft.MagnitudeSpectrum(block, _spectrum, false); break;
case SpectrumType.MagnitudeNormalized:
_getSpectrum = block => _fft.MagnitudeSpectrum(block, _spectrum, true); break;
case SpectrumType.PowerNormalized:
_getSpectrum = block => _fft.PowerSpectrum(block, _spectrum, true); break;
default:
_getSpectrum = block => _fft.PowerSpectrum(block, _spectrum, false); break;
}
// reserve memory for reusable blocks
_spectrum = new float[_blockSize / 2 + 1];
_bandSpectrum = new float[filterbankSize];
}
private void buttonCompute_Click(object sender, EventArgs e)
{
var filterCount = int.Parse(textBoxSize.Text);
var samplingRate = _signal.SamplingRate;
var fftSize = int.Parse(textBoxFftSize.Text);
var lowFreq = float.Parse(textBoxLowFreq.Text);
var highFreq = float.Parse(textBoxHighFreq.Text);
Tuple <double, double, double>[] bands;
float[][] filterbank = null;
VtlnWarper vtln = null;
if (checkBoxVtln.Checked)
{
var alpha = float.Parse(textBoxVtlnAlpha.Text);
var vtlnLow = float.Parse(textBoxVtlnLow.Text);
var vtlnHigh = float.Parse(textBoxVtlnHigh.Text);
vtln = new VtlnWarper(alpha, lowFreq, highFreq, vtlnLow, vtlnHigh);
}
switch (comboBoxFilterbank.Text)
{
case "Mel":
bands = FilterBanks.MelBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, checkBoxOverlap.Checked);
break;
case "Mel Slaney":
bands = FilterBanks.MelBandsSlaney(filterCount, fftSize, samplingRate, lowFreq, highFreq, checkBoxOverlap.Checked);
filterbank = FilterBanks.MelBankSlaney(filterCount, fftSize, samplingRate, lowFreq, highFreq, checkBoxNormalize.Checked, vtln);
break;
case "Bark":
bands = FilterBanks.BarkBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, checkBoxOverlap.Checked);
break;
case "Bark Slaney":
bands = FilterBanks.BarkBandsSlaney(filterCount, fftSize, samplingRate, lowFreq, highFreq, checkBoxOverlap.Checked);
filterbank = FilterBanks.BarkBankSlaney(filterCount, fftSize, samplingRate, lowFreq, highFreq);
break;
case "Critical bands":
bands = FilterBanks.CriticalBands(filterCount, fftSize, samplingRate, lowFreq, highFreq);
break;
case "Octave bands":
bands = FilterBanks.OctaveBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, checkBoxOverlap.Checked);
break;
case "ERB":
bands = null;
filterbank = FilterBanks.Erb(filterCount, fftSize, samplingRate, lowFreq, highFreq);
break;
default:
bands = FilterBanks.HerzBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, checkBoxOverlap.Checked);
break;
}
if (bands != null && filterbank == null)
{
switch (comboBoxShape.Text)
{
case "Triangular":
filterbank = FilterBanks.Triangular(fftSize, samplingRate, bands, vtln, Utils.Scale.HerzToMel);
break;
case "Trapezoidal":
filterbank = FilterBanks.Trapezoidal(fftSize, samplingRate, bands, vtln);
break;
case "BiQuad":
filterbank = FilterBanks.BiQuad(fftSize, samplingRate, bands);
break;
default:
filterbank = FilterBanks.Rectangular(fftSize, samplingRate, bands, vtln);
break;
}
if (checkBoxNormalize.Checked)
{
FilterBanks.Normalize(filterCount, bands, filterbank);
}
}
var spectrumType = (SpectrumType)comboBoxSpectrum.SelectedIndex;
var nonLinearity = (NonLinearityType)comboBoxNonLinearity.SelectedIndex;
var logFloor = float.Parse(textBoxLogFloor.Text);
var mfccExtractor = new MfccExtractor(//samplingRate, 13, 0.025, 0.01,
samplingRate, 13, 512.0 / samplingRate, 0.01,
filterbank: filterbank,
//filterbankSize: 26,
//highFreq: 8000,
//preEmphasis: 0.97,
//lifterSize: 22,
//includeEnergy: true,
spectrumType: spectrumType,
nonLinearity: nonLinearity,
dctType: comboBoxDct.Text,
window: WindowTypes.Hamming,
logFloor: logFloor);
_mfccVectors = mfccExtractor.ComputeFrom(_signal);
//_mfccVectors = mfccExtractor.ComputeFrom(_signal * 32768);
//var mfccVectorsP = mfccExtractor.ParallelComputeFrom(_signal * 32768);
//for (var i = 0; i < _mfccVectors.Count; i++)
//{
// for (var j = 0; j < _mfccVectors[i].Features.Length; j++)
// {
// if (Math.Abs(_mfccVectors[i].Features[j] - mfccVectorsP[i].Features[j]) > 1e-32f)
// {
// MessageBox.Show($"Nope: {i} - {j}");
// return;
// }
// if (Math.Abs(_mfccVectors[i].TimePosition - mfccVectorsP[i].TimePosition) > 1e-32f)
// {
// MessageBox.Show($"Time: {i} - {j}");
// return;
// }
// }
//}
//FeaturePostProcessing.NormalizeMean(_mfccVectors); // optional (but REQUIRED for PNCC!)
//FeaturePostProcessing.AddDeltas(_mfccVectors);
var header = mfccExtractor.FeatureDescriptions;
//.Concat(mfccExtractor.DeltaFeatureDescriptions)
//.Concat(mfccExtractor.DeltaDeltaFeatureDescriptions);
FillFeaturesList(_mfccVectors, header);
mfccListView.Items[0].Selected = true;
melFilterBankPanel.Groups = mfccExtractor.FilterBank;
mfccPanel.Line = _mfccVectors[0].Features;
}
private void filterbankButton_Click(object sender, EventArgs e)
{
var filterCount = int.Parse(filterCountTextBox.Text);
var samplingRate = int.Parse(samplingRateTextBox.Text);
var fftSize = int.Parse(fftSizeTextBox.Text);
var lowFreq = float.Parse(lowFreqTextBox.Text);
var highFreq = float.Parse(highFreqTextBox.Text);
Tuple <double, double, double>[] bands;
switch (filterbankComboBox.Text)
{
case "Mel":
bands = FilterBanks.MelBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, overlapCheckBox.Checked);
break;
case "Bark":
bands = FilterBanks.BarkBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, overlapCheckBox.Checked);
break;
case "Critical bands":
bands = FilterBanks.CriticalBands(filterCount, fftSize, samplingRate, lowFreq, highFreq);
break;
case "Octave bands":
bands = FilterBanks.OctaveBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, overlapCheckBox.Checked);
break;
case "ERB":
bands = null;
_filterbank = FilterBanks.Erb(filterCount, fftSize, samplingRate, lowFreq, highFreq);
// ====================================================
// =================== ! SQUARE ! ====================
//foreach (var filter in _filterbank)
//{
// for (var j = 0; j < filter.Length; j++)
// {
// var squared = filter[j] * filter[j];
// filter[j] = squared;
// }
//}
// normalization coefficient (for plotting)
var scaleCoeff = (int)(1.0 / _filterbank.Max(f => f.Max()));
filterbankPanel.Gain = 100 * scaleCoeff;
break;
default:
bands = FilterBanks.HerzBands(filterCount, fftSize, samplingRate, lowFreq, highFreq, overlapCheckBox.Checked);
break;
}
if (bands != null)
{
switch (shapeComboBox.Text)
{
case "Triangular":
_filterbank = FilterBanks.Triangular(fftSize, samplingRate, bands);
break;
case "Trapezoidal":
_filterbank = FilterBanks.Trapezoidal(fftSize, samplingRate, bands);
break;
case "BiQuad":
_filterbank = FilterBanks.BiQuad(fftSize, samplingRate, bands);
break;
default:
_filterbank = FilterBanks.Rectangular(fftSize, samplingRate, bands);
break;
}
}
band1ComboBox.DataSource = Enumerable.Range(1, filterCount).ToArray();
band2ComboBox.DataSource = Enumerable.Range(1, filterCount).ToArray();
band3ComboBox.DataSource = Enumerable.Range(1, filterCount).ToArray();
band4ComboBox.DataSource = Enumerable.Range(1, filterCount).ToArray();
band1ComboBox.Text = "1";
band2ComboBox.Text = "2";
band3ComboBox.Text = "3";
band4ComboBox.Text = "4";
filterbankPanel.Groups = _filterbank;
}