//##################################################################################################################################
/// <summary>
/// NOTE!!!! The decibel array has been normalised in 0 - 1.
/// </summary>
protected static Tuple <double[, ], double[]> MakeCepstrogram(SonogramConfig config, double[,] matrix, double[] decibels, int sampleRate)
{
double[,] m = matrix;
int nyquist = sampleRate / 2;
double epsilon = config.epsilon;
bool includeDelta = config.mfccConfig.IncludeDelta;
bool includeDoubleDelta = config.mfccConfig.IncludeDoubleDelta;
//Log.WriteIfVerbose(" MakeCepstrogram(matrix, decibels, includeDelta=" + includeDelta + ", includeDoubleDelta=" + includeDoubleDelta + ")");
//(i) APPLY FILTER BANK
int bandCount = config.mfccConfig.FilterbankCount;
bool doMelScale = config.mfccConfig.DoMelScale;
int ccCount = config.mfccConfig.CcCount;
int fftBinCount = config.FreqBinCount; //number of Hz bands = 2^N +1. Subtract DC bin
int minHz = config.MinFreqBand ?? 0;
int maxHz = config.MaxFreqBand ?? nyquist;
Log.WriteIfVerbose("ApplyFilterBank(): Dim prior to filter bank =" + matrix.GetLength(1));
//error check that filterBankCount < FFTbins
if (bandCount > fftBinCount)
{
throw new Exception(
"## FATAL ERROR in BaseSonogram.MakeCepstrogram():- Can't calculate cepstral coeff. FilterbankCount > FFTbins. (" +
bandCount + " > " + fftBinCount + ")\n\n");
}
//this is the filter count for full bandwidth 0-Nyquist. This number is trimmed proportionately to fit the required bandwidth.
if (doMelScale)
{
m = MFCCStuff.MelFilterBank(m, bandCount, nyquist, minHz, maxHz); // using the Greg integral
}
else
{
m = MFCCStuff.LinearFilterBank(m, bandCount, nyquist, minHz, maxHz);
}
Log.WriteIfVerbose("\tDim after filter bank=" + m.GetLength(1) + " (Max filter bank=" + bandCount + ")");
//(ii) CONVERT AMPLITUDES TO DECIBELS
m = MFCCStuff.DecibelSpectra(m, config.WindowPower, sampleRate, epsilon); //from spectrogram
//(iii) NOISE REDUCTION
var tuple1 = SNR.NoiseReduce(m, config.NoiseReductionType, config.NoiseReductionParameter);
m = tuple1.Item1;
//(iv) calculate cepstral coefficients
m = MFCCStuff.Cepstra(m, ccCount);
//(v) NormaliseMatrixValues
m = DataTools.normalise(m);
//(vi) Calculate the full range of MFCC coefficients ie including decibel and deltas, etc
m = MFCCStuff.AcousticVectors(m, decibels, includeDelta, includeDoubleDelta);
var tuple2 = Tuple.Create(m, tuple1.Item2);
return(tuple2); // return matrix and full bandwidth modal noise profile
}
} // Sonogram2ListOfFreqBinArrays()
public static BaseSonogram Audio2DecibelSonogram(FileInfo fiAudio, Dictionary <string, string> configDict)
{
AudioRecording recordingSegment = new AudioRecording(fiAudio.FullName);
SonogramConfig sonoConfig = new SonogramConfig(configDict); //default values config
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
return(sonogram);
}
public SpectrogramStandard(SonogramConfig config, double[,] amplitudeSpectrogram)
: base(config, amplitudeSpectrogram)
{
this.Configuration = config;
this.FrameCount = amplitudeSpectrogram.GetLength(0);
this.Data = amplitudeSpectrogram;
this.Make(this.Data);
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseSonogram"/> class.
/// Use this BASE CONSTRUCTOR when already have the amplitude spectrogram in matrix.
/// Init normalised signal energy array but do nothing with it. This has to be done from outside.
/// </summary>
/// <param name="config">the spectrogram config.</param>
/// <param name="amplitudeSpectrogramData">data of an amplitude Spectrogram.</param>
public BaseSonogram(SonogramConfig config, double[,] amplitudeSpectrogramData)
{
this.Configuration = config;
this.FrameCount = amplitudeSpectrogramData.GetLength(0);
this.SampleRate = this.Configuration.SampleRate;
//init normalised signal energy array but do nothing with it. This has to be done from outside
this.DecibelsNormalised = new double[this.FrameCount];
this.Data = amplitudeSpectrogramData;
}
public static double[,] MakeAcousticVectors(SonogramConfig config, double[,] matrix, double[] decibels, int sampleRate)
{
int ccCount = config.mfccConfig.CcCount;
bool includeDelta = config.mfccConfig.IncludeDelta;
bool includeDoubleDelta = config.mfccConfig.IncludeDoubleDelta;
int deltaT = config.DeltaT;
Log.WriteIfVerbose(" MakeAcousticVectors(matrix, decibels, includeDelta=" + includeDelta + ", includeDoubleDelta=" + includeDoubleDelta + ", deltaT=" + deltaT + ")");
var tuple = MakeCepstrogram(config, matrix, decibels, sampleRate);
double[,] m = tuple.Item1;
//this.SnrData.ModalNoiseProfile = tuple.Item2; //store the full bandwidth modal noise profile
//initialise feature vector for template - will contain three acoustic vectors - for T-dT, T and T+dT
int frameCount = m.GetLength(0);
int cepstralL = m.GetLength(1); // length of cepstral vector
int featurevL = 3 * cepstralL; // to accomodate cepstra for T-2, T and T+2
double[,] acousticM = new double[frameCount, featurevL]; //init the matrix of acoustic vectors
for (int i = deltaT; i < frameCount - deltaT; i++)
{
double[] rowTm2 = DataTools.GetRow(m, i - deltaT);
double[] rowT = DataTools.GetRow(m, i);
double[] rowTp2 = DataTools.GetRow(m, i + deltaT);
for (int j = 0; j < cepstralL; j++)
{
acousticM[i, j] = rowTm2[j];
}
for (int j = 0; j < cepstralL; j++)
{
acousticM[i, cepstralL + j] = rowT[j];
}
for (int j = 0; j < cepstralL; j++)
{
acousticM[i, cepstralL + cepstralL + j] = rowTp2[j];
}
}
return(acousticM);
}
//##################################################################################################################################
//####### STATIC METHODS ###########################################################################################################
//##################################################################################################################################
public static SpectrogramStandard GetSpectralSonogram(string recordingFileName, int frameSize, double windowOverlap, int bitsPerSample, double windowPower, int sr,
TimeSpan duration, NoiseReductionType nrt, double[,] amplitudeSpectrogram)
{
SonogramConfig sonoConfig = new SonogramConfig
{
SourceFName = recordingFileName,
WindowSize = frameSize,
WindowOverlap = windowOverlap,
NoiseReductionType = nrt,
epsilon = Math.Pow(0.5, bitsPerSample - 1),
WindowPower = windowPower,
SampleRate = sr,
Duration = duration,
}; //default values config
var sonogram = new SpectrogramStandard(sonoConfig, amplitudeSpectrogram);
sonogram.SetTimeScale(duration);
return(sonogram);
}
public SpectrogramSobelEdge(string configFile, WavReader wav)
: base(SonogramConfig.Load(configFile), wav)
{
}
public SonogramConfig(SonogramConfig config, Acoustics.Tools.Wav.WavReader wavReader)
{
// TODO: Complete member initialization
this.config = config;
this.wavReader = wavReader;
}
public TriAvSonogram(SonogramConfig config, WavReader wav)
: base(config, wav)
{
}
public SpectrogramSobelEdge(SonogramConfig config, WavReader wav)
: base(config, wav)
{
}
public SpectrogramCepstral(SonogramConfig config, WavReader wav)
: base(config, wav)
{
}
public TriAvSonogram(string configFile, WavReader wav)
: base(SonogramConfig.Load(configFile), wav)
{
}
public static Tuple <SpectrogramStandard, SpectrogramCepstral, double[], double[]> GetAllSonograms(string path, SonogramConfig sonoConfig, int minHz, int maxHz)
{
Log.WriteLine("# Extract spectrogram and cepstrogram from from file: " + Path.GetFileName(path));
var recording = new AudioRecording(path);
// if (recording.SampleRate != 22050) recording.ConvertSampleRate22kHz(); // THIS METHOD CALL IS OBSOLETE
var tuple = GetAllSonograms(recording, sonoConfig, minHz, maxHz);
return(tuple);
}
/// <summary>
/// Returns a Spectrogram and Cepstrogram from the passed recording. These are NOT noise reduced.
/// however, tuple also returns the modal noise and subband modal noise.
/// </summary>
public static Tuple <SpectrogramStandard, SpectrogramCepstral, double[], double[]> GetAllSonograms(AudioRecording recording, SonogramConfig sonoConfig, int minHz, int maxHz)
{
int sr = recording.SampleRate;
bool doMelScale = sonoConfig.DoMelScale;
int ccCount = sonoConfig.mfccConfig.CcCount;
bool includeDelta = sonoConfig.mfccConfig.IncludeDelta;
bool includeDoubleDelta = sonoConfig.mfccConfig.IncludeDoubleDelta;
sonoConfig.SourceFName = recording.BaseName;
var basegram = new AmplitudeSonogram(sonoConfig, recording.WavReader);
var sonogram = new SpectrogramStandard(basegram); //spectrogram has dim[N,257]
Log.WriteLine("Signal: Duration={0}, Sample Rate={1}", sonogram.Duration, sr);
Log.WriteLine(
$"Frames: Size={0}, Count={1}, Duration={2:f1}ms, Overlap={5:f0}%, Offset={3:f1}ms, Frames/s={4:f1}",
sonogram.Configuration.WindowSize, sonogram.FrameCount, sonogram.FrameDuration * 1000,
sonogram.FrameStep * 1000, sonogram.FramesPerSecond, sonoConfig.WindowOverlap * 100);
int binCount = (int)(maxHz / sonogram.FBinWidth) - (int)(minHz / sonogram.FBinWidth) + 1;
Log.WriteLine("Freqs : {0} Hz - {1} Hz. (Freq bin count = {2})", minHz, maxHz, binCount);
Log.WriteLine("MFCCs : doMelScale=" + doMelScale + "; ccCount=" + ccCount + "; includeDelta=" + includeDelta + "; includeDoubleDelta=" + includeDoubleDelta);
//CALCULATE MODAL NOISE PROFILE - USER MAY REQUIRE IT FOR NOISE REDUCTION
double[] modalNoise = sonogram.SnrData.ModalNoiseProfile;
//extract subband modal noise profile
double[] noiseSubband = SpectrogramTools.ExtractModalNoiseSubband(modalNoise, minHz, maxHz, doMelScale,
sonogram.NyquistFrequency, sonogram.FBinWidth);
// CALCULATE CEPSTROGRAM
Log.WriteLine("# Extracting Cepstrogram");
var cepstrogram = new SpectrogramCepstral(basegram, minHz, maxHz); //cepstrogram has dim[N,13]
var tuple = Tuple.Create(sonogram, cepstrogram, modalNoise, noiseSubband);
return(tuple);
}
public SpectrogramStandard(SonogramConfig config, WavReader wav)
: base(config, wav)
{
}
public SpectrogramCepstral(string configFile, WavReader wav)
: this(SonogramConfig.Load(configFile), wav)
{
}
//There are five CONSTRUCTORS
/// <summary>
/// Initializes a new instance of the <see cref="SpectrogramStandard"/> class.
/// Use this constructor when you want to init a new Spectrogram but add the data later.
/// Useful for when constructing artificial spectrograms.
/// </summary>
public SpectrogramStandard(SonogramConfig config)
: base(config)
{
}
public AmplitudeSonogram(SonogramConfig config, WavReader wav)
: base(config, wav, false)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseSonogram"/> class.
/// This BASE CONSTRUCTOR is the one most used - it automatically makes the Amplitude spectrum and
/// then, using a call to Make(), it converts the Amplitude matrix to a Spectrogram whose values are decibels.
/// </summary>
/// <param name="config">All parameters required to make spectrogram.</param>
/// <param name="wav">The recording whose spectrogram is to be made.</param>
public BaseSonogram(SonogramConfig config, WavReader wav)
: this(config, wav, false)
{
this.Make(this.Data);
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseSonogram"/> class.
/// BASE CONSTRUCTOR
/// This constructor contains all steps required to prepare the amplitude spectrogram.
/// The third boolean parameter is simply a place-filler to ensure a different Constructor signature.
/// from the principle Constructor which follows.
/// </summary>
/// <param name="config">config file to use.</param>
/// <param name="wav">wav.</param>
/// <param name="dummy">filler boolean. Calculate in method.</param>
public BaseSonogram(SonogramConfig config, WavReader wav, bool dummy)
: this(config)
{
// As of 28 March 2017 drop capability to get sub-band of spectrogram because was not being used.
// can be recovered later if desired.
//bool doExtractSubband = this.SubBandMinHz > 0 || this.SubBandMaxHz < this.NyquistFrequency;
this.Duration = wav.Time;
double minDuration = 0.2;
if (this.Duration.TotalSeconds < minDuration)
{
LoggedConsole.WriteLine("Signal must at least {0} seconds long to produce a sonogram!", minDuration);
return;
}
//set config params to the current recording
this.SampleRate = wav.SampleRate;
this.Configuration.Duration = wav.Time;
this.Configuration.SampleRate = wav.SampleRate; //also set the Nyquist
this.MaxAmplitude = wav.CalculateMaximumAmplitude();
var recording = new AudioRecording(wav);
var fftData = DSP_Frames.ExtractEnvelopeAndFfts(
recording,
config.WindowSize,
config.WindowOverlap,
this.Configuration.WindowFunction);
// now recover required data
//epsilon is a signal dependent minimum amplitude value to prevent possible subsequent log of zero value.
this.Configuration.epsilon = fftData.Epsilon;
this.Configuration.WindowPower = fftData.WindowPower;
this.FrameCount = fftData.FrameCount;
this.DecibelsPerFrame = fftData.FrameDecibels;
//init normalised signal energy array but do nothing with it. This has to be done from outside
this.DecibelsNormalised = new double[this.FrameCount];
this.Data = fftData.AmplitudeSpectrogram;
// ENERGY PER FRAME and NORMALISED dB PER FRAME AND SNR
// currently DoSnr = true by default
if (config.DoSnr)
{
// If the FractionOfHighEnergyFrames PRIOR to noise removal exceeds SNR.FractionalBoundForMode,
// then Lamel's noise removal algorithm may not work well.
if (fftData.FractionOfHighEnergyFrames > SNR.FractionalBoundForMode)
{
Log.WriteIfVerbose("\nWARNING ##############");
Log.WriteIfVerbose(
"\t############### BaseSonogram(): This is a high energy recording. Percent of high energy frames = {0:f0} > {1:f0}%",
fftData.FractionOfHighEnergyFrames * 100,
SNR.FractionalBoundForMode * 100);
Log.WriteIfVerbose("\t############### Noise reduction algorithm may not work well in this instance!\n");
}
//AUDIO SEGMENTATION/END POINT DETECTION - based on Lamel et al
// Setting segmentation/endpoint detection parameters is broken as of September 2014.
// The next line is a hack replacement
EndpointDetectionConfiguration.SetDefaultSegmentationConfig();
this.SigState = EndpointDetectionConfiguration.DetermineVocalisationEndpoints(this.DecibelsPerFrame, this.FrameStep);
}
/* AS OF 30 MARCH 2017, NO LONGER IMPLEMENT SUB-BAND THINGS, because not being used for years.
* // EXTRACT REQUIRED FREQUENCY BAND
* if (doExtractSubband)
* {
* this.Data = SpectrogramTools.ExtractFreqSubband(
* this.Data,
* this.subBandMinHz,
* this.subBandMaxHz,
* this.Configuration.DoMelScale,
* this.Configuration.FreqBinCount,
* this.FBinWidth);
* this.CalculateSubbandSNR(this.Data);
* }
*/
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseSonogram"/> class.
/// BASE CONSTRUCTOR: Use this when want to extract time segment of existing sonogram.
/// </summary>
/// <param name="config">config file to use.</param>
public BaseSonogram(SonogramConfig config)
{
this.Configuration = config;
this.Duration = config.Duration;
}
