/// <summary>
/// (2) LOWEST PERCENTILE FRAMES METHOD
/// Assumes the passed matrix is a spectrogram and that all values in the data matrix are positive.
/// Returns the noise profile over freq bins. i.e. one noise value per freq bin.
/// First calculate the frame averages, sort in ascending order and accumulate the first N% of frames.
/// WARNING: This method should NOT be used for short recordings i.e LT approx 10-15 seconds long.
/// </summary>
/// <param name="matrix">the spectrogram whose rows=frames, cols=freq bins.</param>
/// <param name="lowPercentile">The percent of lowest energy frames to be included in calculation of the noise profile.</param>
/// <returns>Returns a noise profile consisting of averaged values from the selected X% of low energy frames.</returns>
public static double[] GetNoiseProfile_fromLowestPercentileFrames(double[,] matrix, int lowPercentile)
{
int rowCount = matrix.GetLength(0);
int colCount = matrix.GetLength(1);
int cutoff = lowPercentile * rowCount / 100;
if (cutoff == 0)
{
throw new Exception("Illegal zero value for cutoff in method NoiseRemoval_Briggs.GetNoiseProfile_LowestPercentile()");
}
double[] frameEnergyLevels = MatrixTools.GetRowAverages(matrix);
var sorted = DataTools.SortArrayInAscendingOrder(frameEnergyLevels);
int[] order = sorted.Item1;
// sum the lowest percentile frames
double[] noiseProfile = new double[colCount];
for (int i = 0; i < cutoff; i++)
{
double[] row = DataTools.GetRow(matrix, order[i]);
for (int c = 0; c < colCount; c++)
{
noiseProfile[c] += row[c];
}
}
// get average of the lowest percentile frames
for (int c = 0; c < colCount; c++)
{
noiseProfile[c] /= cutoff;
}
return(noiseProfile);
}
public static double[] GetAvSpectrum_HighestPercentile(double[,] matrix, int highPercentile)
{
double[] energyLevels = MatrixTools.GetRowAverages(matrix);
var sorted = DataTools.SortArray(energyLevels); // sorts array in descending order
int[] order = sorted.Item1;
int colCount = matrix.GetLength(1);
int cutoff = (int)(highPercentile * matrix.GetLength(0) / 100D);
double[] avSpectrum = new double[colCount];
// sum the lowest percentile frames
for (int i = 0; i < cutoff; i++)
{
double[] row = DataTools.GetRow(matrix, order[i]);
for (int c = 0; c < colCount; c++)
{
avSpectrum[c] += row[c];
}
}
// get average of the lowest percentile frames
for (int c = 0; c < colCount; c++)
{
avSpectrum[c] /= cutoff;
}
return(avSpectrum);
}
} //Analysis()
public static Tuple <List <Dictionary <string, double> >, double[]> DetectGratingEvents(double[,] matrix, int colStep, double intensityThreshold)
{
bool doNoiseremoval = true;
int minPeriod = 2; //both period values must be even numbers
int maxPeriod = 20; //Note: 17.2 frames per second i.e. period=20 is just over 1s.
int numberOfCycles = 4;
int step = 1;
int rowCount = matrix.GetLength(0);
int colCount = matrix.GetLength(1);
int numberOfColSteps = colCount / colStep;
var events2return = new List <Dictionary <string, double> >();
double[] array2return = null;
for (int b = 0; b < numberOfColSteps; b++)
{
int minCol = (b * colStep);
int maxCol = minCol + colStep - 1;
double[,] subMatrix = MatrixTools.Submatrix(matrix, 0, minCol, (rowCount - 1), maxCol);
double[] amplitudeArray = MatrixTools.GetRowAverages(subMatrix);
if (doNoiseremoval)
{
double StandardDeviationCount = 0.1; // number of noise SDs to calculate noise threshold - determines severity of noise reduction
SNR.BackgroundNoise bgn = SNR.SubtractBackgroundNoiseFromSignal(amplitudeArray, StandardDeviationCount);
amplitudeArray = bgn.NoiseReducedSignal;
}
//var events = CrossCorrelation.DetectBarsEventsBySegmentationAndXcorrelation(amplitudeArray, intensityThreshold);
var scores = Gratings.ScanArrayForGratingPattern(amplitudeArray, minPeriod, maxPeriod, numberOfCycles, step);
var mergedOutput = Gratings.MergePeriodicScoreArrays(scores, minPeriod, maxPeriod);
double[] intensity = mergedOutput.Item1;
double[] periodicity = mergedOutput.Item2;
var events = Gratings.ExtractPeriodicEvents(intensity, periodicity, intensityThreshold);
foreach (Dictionary <string, double> item in events)
{
item[key_MIN_FREQBIN] = minCol;
item[key_MAX_FREQBIN] = maxCol;
events2return.Add(item);
}
if (b == 3)
{
array2return = amplitudeArray; //returned for debugging purposes only
}
} //for loop over bands of columns
return(Tuple.Create(events2return, array2return));
}//end DetectGratingEvents()
/// <summary>
/// Calculates the average amplitude in the frequency just above the whistle.
/// If it contains above threshold acoustic content, this is unlikely to be a whistle.
/// </summary>
/// <param name="ev">The event.</param>
/// <param name="sonogramData">The spectrogram data as matrix with origin top/left.</param>
/// <param name="bufferBins">THe badnwidth of the buffer zone in bins.</param>
/// <param name="converter">A converter to convert seconds/Hertz to frames/bins.</param>
/// <returns>Average of the spectrogram amplitude in buffer band above whistler.</returns>
public static double GetAverageAmplitudeInNeighbourhood(SpectralEvent ev, double[,] sonogramData, int bufferBins, UnitConverters converter)
{
var bottomBufferBin = converter.GetFreqBinFromHertz(ev.HighFrequencyHertz) + 5;
var topBufferBin = bottomBufferBin + bufferBins;
var frameStart = converter.FrameFromStartTime(ev.EventStartSeconds);
var frameEnd = converter.FrameFromStartTime(ev.EventEndSeconds);
var subMatrix = MatrixTools.Submatrix <double>(sonogramData, frameStart, bottomBufferBin, frameEnd, topBufferBin);
var averageRowDecibels = MatrixTools.GetRowAverages(subMatrix);
var av = averageRowDecibels.Average();
return(av);
}
} //DetectBarsInTheRowsOfaMatrix()
/// <summary>
/// A METHOD TO DETECT HARMONICS IN THE ROWS of the passed portion of a sonogram.
/// This method assume the matrix is derived from a spectrogram rotated so that the matrix rows are spectral columns of sonogram.
/// Was first developed for crow calls.
/// First looks for a decibel profile that matches the passed call duration and decibel loudness.
/// Then samples the centre portion for the correct harmonic period.
/// </summary>
/// <param name="m">data matrix.</param>
/// <param name="dBThreshold">Minimum sound level.</param>
/// <param name="callSpan">Minimum length of call of interest.</param>
/// <returns>a tuple.</returns>
public static Tuple <double[], double[], double[]> DetectHarmonicsInSonogramMatrix(double[,] m, double dBThreshold, int callSpan)
{
int rowCount = m.GetLength(0);
int colCount = m.GetLength(1);
var intensity = new double[rowCount]; //an array of period intensity
var periodicity = new double[rowCount]; //an array of the periodicity values
double[] dBArray = MatrixTools.GetRowAverages(m);
dBArray = DataTools.filterMovingAverage(dBArray, 3);
// for all time frames
for (int t = 0; t < rowCount; t++)
{
if (dBArray[t] < dBThreshold)
{
continue;
}
var row = MatrixTools.GetRow(m, t);
var spectrum = AutoAndCrossCorrelation.CrossCorr(row, row);
int zeroBinCount = 3; //to remove low freq content which dominates the spectrum
for (int s = 0; s < zeroBinCount; s++)
{
spectrum[s] = 0.0; //in real data these bins are dominant and hide other frequency content
}
spectrum = DataTools.NormaliseArea(spectrum);
int maxId = DataTools.GetMaxIndex(spectrum);
double intensityValue = spectrum[maxId];
intensity[t] = intensityValue;
double period = 0.0;
if (maxId != 0)
{
period = 2 * colCount / (double)maxId;
}
periodicity[t] = period;
}
return(Tuple.Create(dBArray, intensity, periodicity));
}
/// <summary>
///
/// </summary>
/// <param name="signal"></param>
/// <param name="fftWindowWidth"></param>
/// <param name="wpdLevelNumber"></param>
/// <returns></returns>
public static double[,] GetFrequencyByOscillationsMatrix(double[] signal, int fftWindowWidth, int wpdLevelNumber)
{
// produce spectrogram
int wpdWindowWidth = (int)Math.Pow(2, wpdLevelNumber);
int sampleCount = signal.Length / wpdWindowWidth;
double[,] wpdByTime = new double[wpdWindowWidth, sampleCount];
double[,] freqByOscillationsMatrix = new double[fftWindowWidth, wpdWindowWidth];
// do a WPD over each frequency bin
// accumulate the WPD spectra into a frequency bin by oscillations per second matrix.
//double[,] matrix = Wavelets.GetWPDSpectralSequence(signal, wpdLevelNumber);
double[,] matrix = WaveletPacketDecomposition.GetWPDEnergySequence(signal, wpdLevelNumber);
double[] V = MatrixTools.GetRowAverages(matrix);
return(freqByOscillationsMatrix);
}
/// <summary>
/// THE KEY ANALYSIS METHOD
/// </summary>
/// <param name="recording">
/// The segment Of Source File.
/// </param>
/// <param name="configDict">
/// The config Dict.
/// </param>
/// <param name="value"></param>
/// <returns>
/// The <see cref="LimnodynastesConvexResults"/>.
/// </returns>
internal static LimnodynastesConvexResults Analysis(
Dictionary <string, double[, ]> dictionaryOfHiResSpectralIndices,
AudioRecording recording,
Dictionary <string, string> configDict,
AnalysisSettings analysisSettings,
SegmentSettingsBase segmentSettings)
{
// for Limnodynastes convex, in the D.Stewart CD, there are peaks close to:
//1. 1950 Hz
//2. 1460 hz
//3. 970 hz These are 490 Hz apart.
// for Limnodynastes convex, in the JCU recording, there are peaks close to:
//1. 1780 Hz
//2. 1330 hz
//3. 880 hz These are 450 Hz apart.
// So strategy is to look for three peaks separated by same amount and in the vicinity of the above,
// starting with highest power (the top peak) and working down to lowest power (bottom peak).
var outputDir = segmentSettings.SegmentOutputDirectory;
TimeSpan segmentStartOffset = segmentSettings.SegmentStartOffset;
//KeyValuePair<string, double[,]> kvp = dictionaryOfHiResSpectralIndices.First();
var spg = dictionaryOfHiResSpectralIndices["RHZ"];
int rhzRowCount = spg.GetLength(0);
int rhzColCount = spg.GetLength(1);
int sampleRate = recording.SampleRate;
double herzPerBin = sampleRate / 2 / (double)rhzRowCount;
double scoreThreshold = (double?)double.Parse(configDict["EventThreshold"]) ?? 3.0;
int minimumFrequency = (int?)int.Parse(configDict["MinHz"]) ?? 850;
int dominantFrequency = (int?)int.Parse(configDict["DominantFrequency"]) ?? 1850;
// # The Limnodynastes call has three major peaks. The dominant peak is at 1850 or as set above.
// # The second and third peak are at equal gaps below. DominantFreq-gap and DominantFreq-(2*gap);
// # Set the gap in the Config file. Should typically be in range 880 to 970
int peakGapInHerz = (int?)int.Parse(configDict["PeakGap"]) ?? 470;
int F1AndF2Gap = (int)Math.Round(peakGapInHerz / herzPerBin);
//int F1AndF2Gap = 10; // 10 = number of freq bins
int F1AndF3Gap = 2 * F1AndF2Gap;
//int F1AndF3Gap = 20;
int hzBuffer = 250;
int bottomBin = 5;
int dominantBin = (int)Math.Round(dominantFrequency / herzPerBin);
int binBuffer = (int)Math.Round(hzBuffer / herzPerBin);;
int dominantBinMin = dominantBin - binBuffer;
int dominantBinMax = dominantBin + binBuffer;
// freqBin + rowID = binCount - 1;
// therefore: rowID = binCount - freqBin - 1;
int minRowID = rhzRowCount - dominantBinMax - 1;
int maxRowID = rhzRowCount - dominantBinMin - 1;
int bottomRow = rhzRowCount - bottomBin - 1;
var list = new List <Point>();
// loop through all spectra/columns of the hi-res spectrogram.
for (int c = 1; c < rhzColCount - 1; c++)
{
double maxAmplitude = -double.MaxValue;
int idOfRowWithMaxAmplitude = 0;
for (int r = minRowID; r <= bottomRow; r++)
{
if (spg[r, c] > maxAmplitude)
{
maxAmplitude = spg[r, c];
idOfRowWithMaxAmplitude = r;
}
}
if (idOfRowWithMaxAmplitude < minRowID)
{
continue;
}
if (idOfRowWithMaxAmplitude > maxRowID)
{
continue;
}
// want a spectral peak.
if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c - 1])
{
continue;
}
if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c + 1])
{
continue;
}
// peak should exceed thresold amplitude
if (spg[idOfRowWithMaxAmplitude, c] < 3.0)
{
continue;
}
// convert row ID to freq bin ID
int freqBinID = rhzRowCount - idOfRowWithMaxAmplitude - 1;
list.Add(new Point(c, freqBinID));
// we now have a list of potential hits for LimCon. This needs to be filtered.
// Console.WriteLine("Col {0}, Bin {1} ", c, freqBinID);
}
// DEBUG ONLY // ################################ TEMPORARY ################################
// superimpose point on RHZ HiRes spectrogram for debug purposes
bool drawOnHiResSpectrogram = true;
//string filePath = @"G:\SensorNetworks\Output\Frogs\TestOfHiResIndices-2016July\Test\Towsey.HiResIndices\SpectrogramImages\3mile_creek_dam_-_Herveys_Range_1076_248366_20130305_001700_30_0min.CombinedGreyScale.png";
var fileName = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name);
string filePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScale.png";
var debugImage = new FileInfo(filePath);
if (!debugImage.Exists)
{
drawOnHiResSpectrogram = false;
}
if (drawOnHiResSpectrogram)
{
// put red dot where max is
Bitmap bmp = new Bitmap(filePath);
foreach (Point point in list)
{
bmp.SetPixel(point.X + 70, 1911 - point.Y, Color.Red);
}
// mark off every tenth frequency bin
for (int r = 0; r < 26; r++)
{
bmp.SetPixel(68, 1911 - (r * 10), Color.Blue);
bmp.SetPixel(69, 1911 - (r * 10), Color.Blue);
}
// mark off upper bound and lower frequency bound
bmp.SetPixel(69, 1911 - dominantBinMin, Color.Lime);
bmp.SetPixel(69, 1911 - dominantBinMax, Color.Lime);
//bmp.SetPixel(69, 1911 - maxRowID, Color.Lime);
string opFilePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScaleAnnotated.png";
bmp.Save(opFilePath);
}
// END DEBUG ################################ TEMPORARY ################################
// now construct the standard decibel spectrogram WITHOUT noise removal, and look for LimConvex
// get frame parameters for the analysis
double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1);
int frameSize = rhzRowCount * 2;
int frameStep = frameSize; // this default = zero overlap
double frameDurationInSeconds = frameSize / (double)sampleRate;
double frameStepInSeconds = frameStep / (double)sampleRate;
double framesPerSec = 1 / frameStepInSeconds;
//var dspOutput = DSP_Frames.ExtractEnvelopeAndFFTs(recording, frameSize, frameStep);
//// Generate deciBel spectrogram
//double[,] deciBelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput.amplitudeSpectrogram, dspOutput.WindowPower, sampleRate, epsilon);
// i: Init SONOGRAM config
var sonoConfig = new SonogramConfig
{
SourceFName = recording.BaseName,
WindowSize = frameSize,
WindowOverlap = 0.0,
NoiseReductionType = NoiseReductionType.None,
};
// init sonogram
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// remove the DC row of the spectrogram
sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.Data.GetLength(0) - 1, sonogram.Data.GetLength(1) - 1);
//scores.Add(new Plot("Decibels", DataTools.NormaliseMatrixValues(dBArray), ActivityAndCover.DefaultActivityThresholdDb));
//scores.Add(new Plot("Active Frames", DataTools.Bool2Binary(activity.activeFrames), 0.0));
// convert spectral peaks to frequency
//var tuple_DecibelPeaks = SpectrogramTools.HistogramOfSpectralPeaks(deciBelSpectrogram);
//int[] peaksBins = tuple_DecibelPeaks.Item2;
//double[] freqPeaks = new double[peaksBins.Length];
//int binCount = sonogram.Data.GetLength(1);
//for (int i = 1; i < peaksBins.Length; i++) freqPeaks[i] = (lowerBinBound + peaksBins[i]) / (double)nyquistBin;
//scores.Add(new Plot("Max Frequency", freqPeaks, 0.0)); // location of peaks for spectral images
// create new list of LimCon hits in the standard spectrogram.
double timeSpanOfFrameInSeconds = frameSize / (double)sampleRate;
var newList = new List <int[]>();
int lastFrameID = sonogram.Data.GetLength(0) - 1;
int lastBinID = sonogram.Data.GetLength(1) - 1;
foreach (Point point in list)
{
double secondsFromStartOfSegment = (point.X * 0.1) + 0.05; // convert point.Y to center of time-block.
int framesFromStartOfSegment = (int)Math.Round(secondsFromStartOfSegment / timeSpanOfFrameInSeconds);
// location of max point is uncertain, so search in neighbourhood.
// NOTE: sonogram.data matrix is time*freqBin
double maxValue = -double.MaxValue;
int idOfTMax = framesFromStartOfSegment;
int idOfFMax = point.Y;
for (int deltaT = -4; deltaT <= 4; deltaT++)
{
for (int deltaF = -1; deltaF <= 1; deltaF++)
{
int newT = framesFromStartOfSegment + deltaT;
if (newT < 0)
{
newT = 0;
}
else if (newT > lastFrameID)
{
newT = lastFrameID;
}
double value = sonogram.Data[newT, point.Y + deltaF];
if (value > maxValue)
{
maxValue = value;
idOfTMax = framesFromStartOfSegment + deltaT;
idOfFMax = point.Y + deltaF;
}
}
}
// newList.Add(new Point(frameSpan, point.Y));
int[] array = new int[2];
array[0] = idOfTMax;
array[1] = idOfFMax;
newList.Add(array);
}
// Now obtain more of spectrogram to see if have peaks at two other places characteristic of Limnodynastes convex.
// In the D.Stewart CD, there are peaks close to:
//1. 1950 Hz
//2. 1460 hz
//3. 970 hz These are 490 Hz apart.
// For Limnodynastes convex, in the JCU recording, there are peaks close to:
//1. 1780 Hz
//2. 1330 hz
//3. 880 hz These are 450 Hz apart.
// So strategy is to look for three peaks separated by same amount and in the vicinity of the above,
// starting with highest power (the top peak) and working down to lowest power (bottom peak).
//We have found top/highest peak - now find the other two.
int secondDominantFrequency = 1380;
int secondDominantBin = (int)Math.Round(secondDominantFrequency / herzPerBin);
int thirdDominantFrequency = 900;
int thirdDominantBin = (int)Math.Round(thirdDominantFrequency / herzPerBin);
var acousticEvents = new List <AcousticEvent>();
int Tbuffer = 2;
// First extract a sub-matrix.
foreach (int[] array in newList)
{
// NOTE: sonogram.data matrix is time*freqBin
int Tframe = array[0];
int F1bin = array[1];
double[,] subMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - Tbuffer, 0, Tframe + Tbuffer, F1bin);
double F1power = subMatrix[Tbuffer, F1bin];
// convert to vector
var spectrum = MatrixTools.GetColumnAverages(subMatrix);
// use the following code to get estimate of background noise
double[,] powerMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - 3, 10, Tframe + 3, F1bin);
double averagePower = (MatrixTools.GetRowAverages(powerMatrix)).Average();
double score = F1power - averagePower;
// debug - checking what the spectrum looks like.
//for (int i = 0; i < 18; i++)
// spectrum[i] = -100.0;
//DataTools.writeBarGraph(spectrum);
// locate the peaks in lower frequency bands, F2 and F3
bool[] peaks = DataTools.GetPeaks(spectrum);
int F2bin = 0;
double F2power = -200.0; // dB
for (int i = -3; i <= 2; i++)
{
int bin = F1bin - F1AndF2Gap + i;
if ((peaks[bin]) && (F2power < subMatrix[1, bin]))
{
F2bin = bin;
F2power = subMatrix[1, bin];
}
}
if (F2bin == 0)
{
continue;
}
if (F2power == -200.0)
{
continue;
}
score += (F2power - averagePower);
int F3bin = 0;
double F3power = -200.0;
for (int i = -5; i <= 2; i++)
{
int bin = F1bin - F1AndF3Gap + i;
if ((peaks[bin]) && (F3power < subMatrix[1, bin]))
{
F3bin = bin;
F3power = subMatrix[1, bin];
}
}
if (F3bin == 0)
{
continue;
}
if (F3power == -200.0)
{
continue;
}
score += (F3power - averagePower);
score /= 3;
// ignore events where SNR < decibel threshold
if (score < scoreThreshold)
{
continue;
}
// ignore events with wrong power distribution. A good LimnoConvex call has strongest F1 power
if ((F3power > F1power) || (F2power > F1power))
{
continue;
}
//freq Bin ID must be converted back to Matrix row ID
// freqBin + rowID = binCount - 1;
// therefore: rowID = binCount - freqBin - 1;
minRowID = rhzRowCount - F1bin - 2;
maxRowID = rhzRowCount - F3bin - 1;
int F1RowID = rhzRowCount - F1bin - 1;
int F2RowID = rhzRowCount - F2bin - 1;
int F3RowID = rhzRowCount - F3bin - 1;
int maxfreq = dominantFrequency + hzBuffer;
int topBin = (int)Math.Round(maxfreq / herzPerBin);
int frameCount = 4;
double duration = frameCount * frameStepInSeconds;
double startTimeWrtSegment = (Tframe - 2) * frameStepInSeconds;
// Got to here so start initialising an acoustic event
var ae = new AcousticEvent(segmentStartOffset, startTimeWrtSegment, duration, minimumFrequency, maxfreq);
ae.SetTimeAndFreqScales(framesPerSec, herzPerBin);
//var ae = new AcousticEvent(oblong, recording.Nyquist, binCount, frameDurationInSeconds, frameStepInSeconds, frameCount);
//ae.StartOffset = TimeSpan.FromSeconds(Tframe * frameStepInSeconds);
var pointF1 = new Point(2, topBin - F1bin);
var pointF2 = new Point(2, topBin - F2bin);
var pointF3 = new Point(2, topBin - F3bin);
ae.Points = new List <Point>();
ae.Points.Add(pointF1);
ae.Points.Add(pointF2);
ae.Points.Add(pointF3);
//tried using HitElements but did not do what I wanted later on.
//ae.HitElements = new HashSet<Point>();
//ae.HitElements = new SortedSet<Point>();
//ae.HitElements.Add(pointF1);
//ae.HitElements.Add(pointF2);
//ae.HitElements.Add(pointF3);
ae.Score = score;
//ae.MinFreq = Math.Round((topBin - F3bin - 5) * herzPerBin);
//ae.MaxFreq = Math.Round(topBin * herzPerBin);
acousticEvents.Add(ae);
}
// now add in extra common info to the acoustic events
acousticEvents.ForEach(ae =>
{
ae.SpeciesName = configDict[AnalysisKeys.SpeciesName];
ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds;
ae.SegmentDurationSeconds = recording.Duration.TotalSeconds;
ae.Name = abbreviatedName;
ae.BorderColour = Color.Red;
ae.FileName = recording.BaseName;
});
double[] scores = new double[rhzColCount]; // predefinition of score array
double nomalisationConstant = scoreThreshold * 4; // four times the score threshold
double compressionFactor = rhzColCount / (double)sonogram.Data.GetLength(0);
foreach (AcousticEvent ae in acousticEvents)
{
ae.ScoreNormalised = ae.Score / nomalisationConstant;
if (ae.ScoreNormalised > 1.0)
{
ae.ScoreNormalised = 1.0;
}
int frameID = (int)Math.Round(ae.EventStartSeconds / frameDurationInSeconds);
int hiresFrameID = (int)Math.Floor(frameID * compressionFactor);
scores[hiresFrameID] = ae.ScoreNormalised;
}
var plot = new Plot(AnalysisName, scores, scoreThreshold);
// DEBUG ONLY ################################ TEMPORARY ################################
// Draw a standard spectrogram and mark of hites etc.
bool createStandardDebugSpectrogram = true;
var imageDir = new DirectoryInfo(outputDir.FullName + @"\SpectrogramImages");
if (!imageDir.Exists)
{
imageDir.Create();
}
if (createStandardDebugSpectrogram)
{
var fileName2 = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name);
string filePath2 = Path.Combine(imageDir.FullName, fileName + ".Spectrogram.png");
Bitmap sonoBmp = (Bitmap)sonogram.GetImage();
int height = sonoBmp.Height;
foreach (AcousticEvent ae in acousticEvents)
{
ae.DrawEvent(sonoBmp);
//g.DrawRectangle(pen, ob.ColumnLeft, ob.RowTop, ob.ColWidth-1, ob.RowWidth);
//ae.DrawPoint(sonoBmp, ae.HitElements.[0], Color.OrangeRed);
//ae.DrawPoint(sonoBmp, ae.HitElements[1], Color.Yellow);
//ae.DrawPoint(sonoBmp, ae.HitElements[2], Color.Green);
ae.DrawPoint(sonoBmp, ae.Points[0], Color.OrangeRed);
ae.DrawPoint(sonoBmp, ae.Points[1], Color.Yellow);
ae.DrawPoint(sonoBmp, ae.Points[2], Color.LimeGreen);
}
// draw the original hits on the standard sonogram
foreach (int[] array in newList)
{
sonoBmp.SetPixel(array[0], height - array[1], Color.Cyan);
}
// mark off every tenth frequency bin on the standard sonogram
for (int r = 0; r < 20; r++)
{
sonoBmp.SetPixel(0, height - (r * 10) - 1, Color.Blue);
sonoBmp.SetPixel(1, height - (r * 10) - 1, Color.Blue);
}
// mark off upper bound and lower frequency bound
sonoBmp.SetPixel(0, height - dominantBinMin, Color.Lime);
sonoBmp.SetPixel(0, height - dominantBinMax, Color.Lime);
sonoBmp.Save(filePath2);
}
// END DEBUG ################################ TEMPORARY ################################
return(new LimnodynastesConvexResults
{
Sonogram = sonogram,
Hits = null,
Plot = plot,
Events = acousticEvents,
RecordingDuration = recording.Duration,
});
} // Analysis()
/// <summary>
/// Calculate summary statistics for supplied temporal and spectral targets.
/// </summary>
/// <remarks>
/// The acoustic statistics calculated in this method are based on methods outlined in
/// "Acoustic classification of multiple simultaneous bird species: A multi-instance multi-label approach",
/// by Forrest Briggs, Balaji Lakshminarayanan, Lawrence Neal, Xiaoli Z.Fern, Raviv Raich, Sarah J.K.Hadley, Adam S. Hadley, Matthew G. Betts, et al.
/// The Journal of the Acoustical Society of America v131, pp4640 (2012); doi: http://dx.doi.org/10.1121/1.4707424
/// ..
/// The Briggs feature are calculated from the column (freq bin) and row (frame) sums of the extracted spectrogram.
/// 1. Gini Index for frame and bin sums. A measure of dispersion. Problem with gini is that its value is dependent on the row or column count.
/// We use entropy instead because value not dependent on row or column count because it is normalized.
/// For the following meausres of k-central moments, the freq and time values are normalized in 0,1 to width of the event.
/// 2. freq-mean
/// 3. freq-variance
/// 4. freq-skew and kurtosis
/// 5. time-mean
/// 6. time-variance
/// 7. time-skew and kurtosis
/// 8. freq-max (normalized)
/// 9. time-max (normalized)
/// 10. Briggs et al also calculate a 16 value histogram of gradients for each event mask. We do not do that here although we could.
/// ...
/// NOTE 1: There are differences between our method of noise reduction and Briggs. Briggs does not convert to decibels
/// and instead works with power values. He obtains a noise profile from the 20% of frames having the lowest energy sum.
/// NOTE 2: To NormaliseMatrixValues for noise, they divide the actual energy by the noise value. This is equivalent to subtraction when working in decibels.
/// There are advantages and disadvantages to Briggs method versus ours. In our case, we hve to convert decibel values back to
/// energy values when calculating the statistics for the extracted acoustic event.
/// NOTE 3: We do not calculate the higher central moments of the time/frequency profiles, i.e. skew and kurtosis.
/// Ony mean and standard deviation.
/// ..
/// NOTE 4: This method assumes that the passed event occurs totally within the passed recording,
/// AND that the passed recording is of sufficient duration to obtain reliable BGN noise profile
/// BUT not so long as to cause memory constipation.
/// </remarks>
/// <param name="recording">as type AudioRecording which contains the event</param>
/// <param name="temporalTarget">Both start and end bounds - relative to the supplied recording</param>
/// <param name="spectralTarget">both bottom and top bounds in Hertz</param>
/// <param name="config">parameters that determine the outcome of the analysis</param>
/// <param name="segmentStartOffset">How long since the start of the recording this event occurred</param>
/// <returns>an instance of EventStatistics</returns>
public static EventStatistics AnalyzeAudioEvent(
AudioRecording recording,
Range <TimeSpan> temporalTarget,
Range <double> spectralTarget,
EventStatisticsConfiguration config,
TimeSpan segmentStartOffset)
{
var stats = new EventStatistics
{
EventStartSeconds = temporalTarget.Minimum.TotalSeconds,
EventEndSeconds = temporalTarget.Maximum.TotalSeconds,
LowFrequencyHertz = spectralTarget.Minimum,
HighFrequencyHertz = spectralTarget.Maximum,
SegmentDurationSeconds = recording.Duration.TotalSeconds,
SegmentStartSeconds = segmentStartOffset.TotalSeconds,
};
// temporal target is supplied relative to recording, but not the supplied audio segment
// shift coordinates relative to segment
var localTemporalTarget = temporalTarget.Shift(-segmentStartOffset);
if (!recording
.Duration
.AsRangeFromZero(Topology.Inclusive)
.Contains(localTemporalTarget))
{
stats.Error = true;
stats.ErrorMessage =
$"Audio not long enough ({recording.Duration}) to analyze target ({localTemporalTarget})";
return(stats);
}
// convert recording to spectrogram
int sampleRate = recording.SampleRate;
double epsilon = recording.Epsilon;
// extract the spectrogram
var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recording, config.FrameSize, config.FrameStep);
double hertzBinWidth = dspOutput1.FreqBinWidth;
var stepDurationInSeconds = config.FrameStep / (double)sampleRate;
var startFrame = (int)Math.Ceiling(localTemporalTarget.Minimum.TotalSeconds / stepDurationInSeconds);
// subtract 1 frame because want to end before start of end point.
var endFrame = (int)Math.Floor(localTemporalTarget.Maximum.TotalSeconds / stepDurationInSeconds) - 1;
var bottomBin = (int)Math.Floor(spectralTarget.Minimum / hertzBinWidth);
var topBin = (int)Math.Ceiling(spectralTarget.Maximum / hertzBinWidth);
// Events can have their high value set to the nyquist.
// Since the submatrix call below uses an inclusive upper bound an index out of bounds exception occurs in
// these cases. So we just ask for the bin below.
if (topBin >= config.FrameSize / 2)
{
topBin = (config.FrameSize / 2) - 1;
}
// Convert amplitude spectrogram to deciBels and calculate the dB background noise profile
double[,] decibelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput1.AmplitudeSpectrogram, dspOutput1.WindowPower, sampleRate, epsilon);
double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram);
decibelSpectrogram = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram, spectralDecibelBgn);
decibelSpectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram, nhThreshold: 2.0);
// extract the required acoustic event
var eventMatrix = MatrixTools.Submatrix(decibelSpectrogram, startFrame, bottomBin, endFrame, topBin);
// Get the SNR of the event. This is just the max value in the matrix because noise reduced
MatrixTools.MinMax(eventMatrix, out _, out double max);
stats.SnrDecibels = max;
// Now need to convert event matrix back to energy values before calculating other statistics
eventMatrix = MatrixTools.Decibels2Power(eventMatrix);
var columnAverages = MatrixTools.GetColumnAverages(eventMatrix);
var rowAverages = MatrixTools.GetRowAverages(eventMatrix);
// calculate the mean and temporal standard deviation in decibels
NormalDist.AverageAndSD(rowAverages, out double mean, out double stddev);
stats.MeanDecibels = 10 * Math.Log10(mean);
stats.TemporalStdDevDecibels = 10 * Math.Log10(stddev);
// calculate the frequency standard deviation in decibels
NormalDist.AverageAndSD(columnAverages, out mean, out stddev);
stats.FreqBinStdDevDecibels = 10 * Math.Log10(stddev);
// calculate relative location of the temporal maximum
int maxRowId = DataTools.GetMaxIndex(rowAverages);
stats.TemporalMaxRelative = maxRowId / (double)rowAverages.Length;
// calculate the entropy dispersion/concentration indices
stats.TemporalEnergyDistribution = 1 - DataTools.EntropyNormalised(rowAverages);
stats.SpectralEnergyDistribution = 1 - DataTools.EntropyNormalised(columnAverages);
// calculate the spectral centroid and the dominant frequency
double binCentroid = CalculateSpectralCentroid(columnAverages);
stats.SpectralCentroid = (int)Math.Round(hertzBinWidth * binCentroid) + (int)spectralTarget.Minimum;
int maxColumnId = DataTools.GetMaxIndex(columnAverages);
stats.DominantFrequency = (int)Math.Round(hertzBinWidth * maxColumnId) + (int)spectralTarget.Minimum;
// remainder of this method is to produce debugging images. Can comment out when not debugging.
/*
* var normalisedIndex = DataTools.NormaliseMatrixValues(columnAverages);
* var image4 = GraphsAndCharts.DrawGraph("columnSums", normalisedIndex, 100);
* string path4 = @"C:\SensorNetworks\Output\Sonograms\UnitTestSonograms\columnSums.png";
* image4.Save(path4);
* normalisedIndex = DataTools.NormaliseMatrixValues(rowAverages);
* image4 = GraphsAndCharts.DrawGraph("rowSums", normalisedIndex, 100);
* path4 = @"C:\SensorNetworks\Output\Sonograms\UnitTestSonograms\rowSums.png";
* image4.Save(path4);
*/
return(stats);
}
//public const string key_COUNT = "count";
public static Tuple <double[, ], double[, ], double[, ], double[]> DetectBarsUsingXcorrelation(double[,] m, int rowStep, int rowWidth, int colStep, int colWidth,
double intensityThreshold, int zeroBinCount)
{
bool doNoiseremoval = true;
//intensityThreshold = 0.3;
int rowCount = m.GetLength(0);
int colCount = m.GetLength(1);
int numberOfColSteps = colCount / colStep;
int numberOfRowSteps = rowCount / rowStep;
var intensityMatrix = new double[numberOfRowSteps, numberOfColSteps];
var periodicityMatrix = new double[numberOfRowSteps, numberOfColSteps];
var hitsMatrix = new double[rowCount, colCount];
double[] array2return = null;
for (int b = 0; b < numberOfColSteps; b++)
{
int minCol = b * colStep;
int maxCol = minCol + colWidth - 1;
double[,] subMatrix = MatrixTools.Submatrix(m, 0, minCol, rowCount - 1, maxCol);
double[] amplitudeArray = MatrixTools.GetRowAverages(subMatrix);
if (doNoiseremoval)
{
double StandardDeviationCount = 0.1; // number of noise SDs to calculate noise threshold - determines severity of noise reduction
SNR.BackgroundNoise bgn = SNR.SubtractBackgroundNoiseFromSignal(amplitudeArray, StandardDeviationCount);
amplitudeArray = bgn.NoiseReducedSignal;
}
//double noiseThreshold = 0.005;
//for (int i = 1; i < amplitudeArray.Length - 1; i++)
//{
// if ((amplitudeArray[i - 1] < noiseThreshold) && (amplitudeArray[i + 1] < noiseThreshold)) amplitudeArray[i] = 0.0;
//}
//DataTools.writeBarGraph(amplitudeArray);
if (b == 2)
{
array2return = amplitudeArray; //returned for debugging purposes only
}
//ii: DETECT HARMONICS
var results = AutoAndCrossCorrelation.DetectPeriodicityInLongArray(amplitudeArray, rowStep, rowWidth, zeroBinCount);
double[] intensity = results.Item1; //an array of periodicity scores
double[] periodicity = results.Item2;
//transfer periodicity info to a matrices.
for (int rs = 0; rs < numberOfRowSteps; rs++)
{
intensityMatrix[rs, b] = intensity[rs];
periodicityMatrix[rs, b] = periodicity[rs];
//mark up the hits matrix
//double relativePeriod = periodicity[rs] / rowWidth / 2;
if (intensity[rs] > intensityThreshold)
{
int minRow = rs * rowStep;
int maxRow = minRow + rowStep - 1;
for (int r = minRow; r < maxRow; r++)
{
for (int c = minCol; c < maxCol; c++)
{
//hitsMatrix[r, c] = relativePeriod;
hitsMatrix[r, c] = periodicity[rs];
}
}
} // if()
} // for loop over numberOfRowSteps
} // for loop over numberOfColSteps
return(Tuple.Create(intensityMatrix, periodicityMatrix, hitsMatrix, array2return));
}
} //DetectBarsInTheRowsOfaMatrix()
/// A METHOD TO DETECT HARMONICS IN THE ROWS of the passed portion of a sonogram.
/// This method assume the matrix is derived from a spectrogram rotated so that the matrix rows are spectral columns of sonogram.
/// Was first developed for crow calls.
/// First looks for a decibel profile that matches the passed call duration and decibel loudness
/// Then samples the centre portion for the correct harmonic period.
/// </summary>
/// <param name="m"></param>
/// <param name="amplitudeThreshold"></param>
/// <returns></returns>
public static Tuple <double[], double[], double[]> DetectHarmonicsInSonogramMatrix(double[,] m, double dBThreshold, int callSpan)
{
int zeroBinCount = 3; //to remove low freq content which dominates the spectrum
int halfspan = callSpan / 2;
double[] dBArray = MatrixTools.GetRowAverages(m);
dBArray = DataTools.filterMovingAverage(dBArray, 3);
bool doNoiseRemoval = true;
if (doNoiseRemoval)
{
double StandardDeviationCount = 0.1; // number of noise SDs to calculate noise threshold - determines severity of noise reduction
SNR.BackgroundNoise bgn = SNR.SubtractBackgroundNoiseFromSignal(dBArray, StandardDeviationCount);
dBArray = bgn.NoiseReducedSignal;
}
bool[] peaks = DataTools.GetPeaks(dBArray);
int rowCount = m.GetLength(0);
int colCount = m.GetLength(1);
var intensity = new double[rowCount]; //an array of period intensity
var periodicity = new double[rowCount]; //an array of the periodicity values
for (int r = halfspan; r < rowCount - halfspan; r++)
{
//APPLY A FILTER: must satisfy the following conditions for a call.
if (!peaks[r])
{
continue;
}
if (dBArray[r] < dBThreshold)
{
continue;
}
double lowerDiff = dBArray[r] - dBArray[r - halfspan];
double upperDiff = dBArray[r] - dBArray[r + halfspan];
if (lowerDiff < dBThreshold || upperDiff < dBThreshold)
{
continue;
}
double[] prevRow = DataTools.DiffFromMean(MatrixTools.GetRow(m, r - 1));
double[] thisRow = DataTools.DiffFromMean(MatrixTools.GetRow(m, r));
var spectrum = AutoAndCrossCorrelation.CrossCorr(prevRow, thisRow);
for (int s = 0; s < zeroBinCount; s++)
{
spectrum[s] = 0.0; //in real data these bins are dominant and hide other frequency content
}
spectrum = DataTools.NormaliseArea(spectrum);
int maxId = DataTools.GetMaxIndex(spectrum);
double intensityValue = spectrum[maxId];
intensity[r] = intensityValue;
double period = 0.0;
if (maxId != 0)
{
period = 2 * colCount / (double)maxId;
}
periodicity[r] = period;
prevRow = thisRow;
} // rows
return(Tuple.Create(dBArray, intensity, periodicity));
} //DetectHarmonicsInSonogramMatrix()
public static double[,] GetFrequencyByOscillationsMatrix(double[,] spectrogram, double sensitivity, int sampleLength, string algorithmName)
{
int frameCount = spectrogram.GetLength(0);
int freqBinCount = spectrogram.GetLength(1);
var freqByOscMatrix = new double[sampleLength / 2, freqBinCount];
// over all frequency bins
for (int bin = 0; bin < freqBinCount; bin++)
{
double[,] subM;
// get average of three bins
if (bin == 0)
{
subM = MatrixTools.Submatrix(spectrogram, 0, 0, frameCount - 1, 2);
}
else // get average of three bins
if (bin == freqBinCount - 1)
{
subM = MatrixTools.Submatrix(spectrogram, 0, bin - 2, frameCount - 1, bin);
}
else
{
// get average of three bins
subM = MatrixTools.Submatrix(spectrogram, 0, bin - 1, frameCount - 1, bin + 1);
}
var freqBin = MatrixTools.GetRowAverages(subM);
// could alternatively take single bins but averaging three bins seems to work well
//var freqBin = MatrixTools.GetColumn(spectrogram, bin);
// vector to store the oscillations vector derived from one frequency bin.
double[] oscillationsSpectrum = null;
var xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength);
// Use the Autocorrelation - FFT option.
// This option appears to work best for use as a spectral index to detect oscillations
if (algorithmName.Equals("autocorr-fft"))
{
oscillationsSpectrum = GetOscillationArrayUsingFft(xCorrByTimeMatrix, sensitivity);
}
// Use the Autocorrelation - SVD - FFT option.
if (algorithmName.Equals("autocorr-svd-fft"))
{
oscillationsSpectrum = GetOscillationArrayUsingSvdAndFft(xCorrByTimeMatrix, sensitivity, bin);
}
// Use the Wavelet Transform
if (algorithmName.Equals("Autocorr-WPD"))
{
oscillationsSpectrum = GetOscillationArrayUsingWpd(xCorrByTimeMatrix, sensitivity, bin);
//WaveletTransformContinuous cwt = new WaveletTransformContinuous(freqBin, maxScale);
//double[,] cwtMatrix = cwt.GetScaleTimeMatrix();
//oscillationsSpectrum = GetOscillationArrayUsingCWT(cwtMatrix, sensitivity, bin);
//double[] dynamicRanges = GetVectorOfDynamicRanges(freqBin, sampleLength);
}
// transfer final oscillation vector for single frequency bin to the Oscillations by frequency matrix.
MatrixTools.SetColumn(freqByOscMatrix, bin, oscillationsSpectrum);
} // foreach frequency bin
return(freqByOscMatrix);
}
public static double[,] GetFrequencyByOscillationsMatrix(double[,] spectrogram, double sensitivity, int sampleLength, string algorithmName)
{
int frameCount = spectrogram.GetLength(0);
int freqBinCount = spectrogram.GetLength(1);
double[,] freqByOscMatrix = new double[sampleLength / 2, freqBinCount];
// over all frequency bins
for (int bin = 0; bin < freqBinCount; bin++)
{
//bin = 50; // for debugging
//Console.WriteLine("Bin = {0}", bin);
double[,] subM;
// get average of three bins
if (bin == 0)
{
subM = MatrixTools.Submatrix(spectrogram, 0, 0, frameCount - 1, 2);
}
else // get average of three bins
if (bin == freqBinCount - 1)
{
subM = MatrixTools.Submatrix(spectrogram, 0, bin - 2, frameCount - 1, bin);
}
else
{
// get average of three bins
subM = MatrixTools.Submatrix(spectrogram, 0, bin - 1, frameCount - 1, bin + 1);
}
var freqBin = MatrixTools.GetRowAverages(subM);
// vector to store the oscilations vector derived from one frequency bin.
double[] oscillationsSpectrum = null;
// Use the Autocorrelation - SVD - FFT option.
if (algorithmName.Equals("autocorr-svd-fft"))
{
double[,] xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength);
//xcorCount += xCorrByTimeMatrix.GetLength(1);
oscillationsSpectrum = GetOscillationArrayUsingSvdAndFft(xCorrByTimeMatrix, sensitivity, bin);
}
// Use the Autocorrelation - FFT option.
if (algorithmName.Equals("autocorr-fft"))
{
double[,] xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength);
oscillationsSpectrum = GetOscillationArrayUsingFft(xCorrByTimeMatrix, sensitivity, bin);
}
// Use the Wavelet Transform
if (algorithmName.Equals("Autocorr-WPD"))
{
double[,] xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength);
oscillationsSpectrum = GetOscillationArrayUsingWpd(xCorrByTimeMatrix, sensitivity, bin);
//WaveletTransformContinuous cwt = new WaveletTransformContinuous(freqBin, maxScale);
//double[,] cwtMatrix = cwt.GetScaleTimeMatrix();
//oscillationsSpectrum = GetOscillationArrayUsingCWT(cwtMatrix, sensitivity, bin);
//double[] dynamicRanges = GetVectorOfDynamicRanges(freqBin, sampleLength);
}
// transfer final oscillation vector to the Oscillations by frequency matrix.
MatrixTools.SetColumn(freqByOscMatrix, bin, oscillationsSpectrum);
} // feareach frequency bin
return(freqByOscMatrix);
}
