/// <summary> /// (2) LOWEST PERCENTILE FRAMES METHOD /// Assumes the passed matrix is a spectrogram and that all values in the data matrix are positive. /// Returns the noise profile over freq bins. i.e. one noise value per freq bin. /// First calculate the frame averages, sort in ascending order and accumulate the first N% of frames. /// WARNING: This method should NOT be used for short recordings i.e LT approx 10-15 seconds long. /// </summary> /// <param name="matrix">the spectrogram whose rows=frames, cols=freq bins.</param> /// <param name="lowPercentile">The percent of lowest energy frames to be included in calculation of the noise profile.</param> /// <returns>Returns a noise profile consisting of averaged values from the selected X% of low energy frames.</returns> public static double[] GetNoiseProfile_fromLowestPercentileFrames(double[,] matrix, int lowPercentile) { int rowCount = matrix.GetLength(0); int colCount = matrix.GetLength(1); int cutoff = lowPercentile * rowCount / 100; if (cutoff == 0) { throw new Exception("Illegal zero value for cutoff in method NoiseRemoval_Briggs.GetNoiseProfile_LowestPercentile()"); } double[] frameEnergyLevels = MatrixTools.GetRowAverages(matrix); var sorted = DataTools.SortArrayInAscendingOrder(frameEnergyLevels); int[] order = sorted.Item1; // sum the lowest percentile frames double[] noiseProfile = new double[colCount]; for (int i = 0; i < cutoff; i++) { double[] row = DataTools.GetRow(matrix, order[i]); for (int c = 0; c < colCount; c++) { noiseProfile[c] += row[c]; } } // get average of the lowest percentile frames for (int c = 0; c < colCount; c++) { noiseProfile[c] /= cutoff; } return(noiseProfile); }
public static double[] GetAvSpectrum_HighestPercentile(double[,] matrix, int highPercentile) { double[] energyLevels = MatrixTools.GetRowAverages(matrix); var sorted = DataTools.SortArray(energyLevels); // sorts array in descending order int[] order = sorted.Item1; int colCount = matrix.GetLength(1); int cutoff = (int)(highPercentile * matrix.GetLength(0) / 100D); double[] avSpectrum = new double[colCount]; // sum the lowest percentile frames for (int i = 0; i < cutoff; i++) { double[] row = DataTools.GetRow(matrix, order[i]); for (int c = 0; c < colCount; c++) { avSpectrum[c] += row[c]; } } // get average of the lowest percentile frames for (int c = 0; c < colCount; c++) { avSpectrum[c] /= cutoff; } return(avSpectrum); }
} //Analysis() public static Tuple <List <Dictionary <string, double> >, double[]> DetectGratingEvents(double[,] matrix, int colStep, double intensityThreshold) { bool doNoiseremoval = true; int minPeriod = 2; //both period values must be even numbers int maxPeriod = 20; //Note: 17.2 frames per second i.e. period=20 is just over 1s. int numberOfCycles = 4; int step = 1; int rowCount = matrix.GetLength(0); int colCount = matrix.GetLength(1); int numberOfColSteps = colCount / colStep; var events2return = new List <Dictionary <string, double> >(); double[] array2return = null; for (int b = 0; b < numberOfColSteps; b++) { int minCol = (b * colStep); int maxCol = minCol + colStep - 1; double[,] subMatrix = MatrixTools.Submatrix(matrix, 0, minCol, (rowCount - 1), maxCol); double[] amplitudeArray = MatrixTools.GetRowAverages(subMatrix); if (doNoiseremoval) { double StandardDeviationCount = 0.1; // number of noise SDs to calculate noise threshold - determines severity of noise reduction SNR.BackgroundNoise bgn = SNR.SubtractBackgroundNoiseFromSignal(amplitudeArray, StandardDeviationCount); amplitudeArray = bgn.NoiseReducedSignal; } //var events = CrossCorrelation.DetectBarsEventsBySegmentationAndXcorrelation(amplitudeArray, intensityThreshold); var scores = Gratings.ScanArrayForGratingPattern(amplitudeArray, minPeriod, maxPeriod, numberOfCycles, step); var mergedOutput = Gratings.MergePeriodicScoreArrays(scores, minPeriod, maxPeriod); double[] intensity = mergedOutput.Item1; double[] periodicity = mergedOutput.Item2; var events = Gratings.ExtractPeriodicEvents(intensity, periodicity, intensityThreshold); foreach (Dictionary <string, double> item in events) { item[key_MIN_FREQBIN] = minCol; item[key_MAX_FREQBIN] = maxCol; events2return.Add(item); } if (b == 3) { array2return = amplitudeArray; //returned for debugging purposes only } } //for loop over bands of columns return(Tuple.Create(events2return, array2return)); }//end DetectGratingEvents()
/// <summary> /// Calculates the average amplitude in the frequency just above the whistle. /// If it contains above threshold acoustic content, this is unlikely to be a whistle. /// </summary> /// <param name="ev">The event.</param> /// <param name="sonogramData">The spectrogram data as matrix with origin top/left.</param> /// <param name="bufferBins">THe badnwidth of the buffer zone in bins.</param> /// <param name="converter">A converter to convert seconds/Hertz to frames/bins.</param> /// <returns>Average of the spectrogram amplitude in buffer band above whistler.</returns> public static double GetAverageAmplitudeInNeighbourhood(SpectralEvent ev, double[,] sonogramData, int bufferBins, UnitConverters converter) { var bottomBufferBin = converter.GetFreqBinFromHertz(ev.HighFrequencyHertz) + 5; var topBufferBin = bottomBufferBin + bufferBins; var frameStart = converter.FrameFromStartTime(ev.EventStartSeconds); var frameEnd = converter.FrameFromStartTime(ev.EventEndSeconds); var subMatrix = MatrixTools.Submatrix <double>(sonogramData, frameStart, bottomBufferBin, frameEnd, topBufferBin); var averageRowDecibels = MatrixTools.GetRowAverages(subMatrix); var av = averageRowDecibels.Average(); return(av); }
} //DetectBarsInTheRowsOfaMatrix() /// <summary> /// A METHOD TO DETECT HARMONICS IN THE ROWS of the passed portion of a sonogram. /// This method assume the matrix is derived from a spectrogram rotated so that the matrix rows are spectral columns of sonogram. /// Was first developed for crow calls. /// First looks for a decibel profile that matches the passed call duration and decibel loudness. /// Then samples the centre portion for the correct harmonic period. /// </summary> /// <param name="m">data matrix.</param> /// <param name="dBThreshold">Minimum sound level.</param> /// <param name="callSpan">Minimum length of call of interest.</param> /// <returns>a tuple.</returns> public static Tuple <double[], double[], double[]> DetectHarmonicsInSonogramMatrix(double[,] m, double dBThreshold, int callSpan) { int rowCount = m.GetLength(0); int colCount = m.GetLength(1); var intensity = new double[rowCount]; //an array of period intensity var periodicity = new double[rowCount]; //an array of the periodicity values double[] dBArray = MatrixTools.GetRowAverages(m); dBArray = DataTools.filterMovingAverage(dBArray, 3); // for all time frames for (int t = 0; t < rowCount; t++) { if (dBArray[t] < dBThreshold) { continue; } var row = MatrixTools.GetRow(m, t); var spectrum = AutoAndCrossCorrelation.CrossCorr(row, row); int zeroBinCount = 3; //to remove low freq content which dominates the spectrum for (int s = 0; s < zeroBinCount; s++) { spectrum[s] = 0.0; //in real data these bins are dominant and hide other frequency content } spectrum = DataTools.NormaliseArea(spectrum); int maxId = DataTools.GetMaxIndex(spectrum); double intensityValue = spectrum[maxId]; intensity[t] = intensityValue; double period = 0.0; if (maxId != 0) { period = 2 * colCount / (double)maxId; } periodicity[t] = period; } return(Tuple.Create(dBArray, intensity, periodicity)); }
/// <summary> /// /// </summary> /// <param name="signal"></param> /// <param name="fftWindowWidth"></param> /// <param name="wpdLevelNumber"></param> /// <returns></returns> public static double[,] GetFrequencyByOscillationsMatrix(double[] signal, int fftWindowWidth, int wpdLevelNumber) { // produce spectrogram int wpdWindowWidth = (int)Math.Pow(2, wpdLevelNumber); int sampleCount = signal.Length / wpdWindowWidth; double[,] wpdByTime = new double[wpdWindowWidth, sampleCount]; double[,] freqByOscillationsMatrix = new double[fftWindowWidth, wpdWindowWidth]; // do a WPD over each frequency bin // accumulate the WPD spectra into a frequency bin by oscillations per second matrix. //double[,] matrix = Wavelets.GetWPDSpectralSequence(signal, wpdLevelNumber); double[,] matrix = WaveletPacketDecomposition.GetWPDEnergySequence(signal, wpdLevelNumber); double[] V = MatrixTools.GetRowAverages(matrix); return(freqByOscillationsMatrix); }
/// <summary> /// THE KEY ANALYSIS METHOD /// </summary> /// <param name="recording"> /// The segment Of Source File. /// </param> /// <param name="configDict"> /// The config Dict. /// </param> /// <param name="value"></param> /// <returns> /// The <see cref="LimnodynastesConvexResults"/>. /// </returns> internal static LimnodynastesConvexResults Analysis( Dictionary <string, double[, ]> dictionaryOfHiResSpectralIndices, AudioRecording recording, Dictionary <string, string> configDict, AnalysisSettings analysisSettings, SegmentSettingsBase segmentSettings) { // for Limnodynastes convex, in the D.Stewart CD, there are peaks close to: //1. 1950 Hz //2. 1460 hz //3. 970 hz These are 490 Hz apart. // for Limnodynastes convex, in the JCU recording, there are peaks close to: //1. 1780 Hz //2. 1330 hz //3. 880 hz These are 450 Hz apart. // So strategy is to look for three peaks separated by same amount and in the vicinity of the above, // starting with highest power (the top peak) and working down to lowest power (bottom peak). var outputDir = segmentSettings.SegmentOutputDirectory; TimeSpan segmentStartOffset = segmentSettings.SegmentStartOffset; //KeyValuePair<string, double[,]> kvp = dictionaryOfHiResSpectralIndices.First(); var spg = dictionaryOfHiResSpectralIndices["RHZ"]; int rhzRowCount = spg.GetLength(0); int rhzColCount = spg.GetLength(1); int sampleRate = recording.SampleRate; double herzPerBin = sampleRate / 2 / (double)rhzRowCount; double scoreThreshold = (double?)double.Parse(configDict["EventThreshold"]) ?? 3.0; int minimumFrequency = (int?)int.Parse(configDict["MinHz"]) ?? 850; int dominantFrequency = (int?)int.Parse(configDict["DominantFrequency"]) ?? 1850; // # The Limnodynastes call has three major peaks. The dominant peak is at 1850 or as set above. // # The second and third peak are at equal gaps below. DominantFreq-gap and DominantFreq-(2*gap); // # Set the gap in the Config file. Should typically be in range 880 to 970 int peakGapInHerz = (int?)int.Parse(configDict["PeakGap"]) ?? 470; int F1AndF2Gap = (int)Math.Round(peakGapInHerz / herzPerBin); //int F1AndF2Gap = 10; // 10 = number of freq bins int F1AndF3Gap = 2 * F1AndF2Gap; //int F1AndF3Gap = 20; int hzBuffer = 250; int bottomBin = 5; int dominantBin = (int)Math.Round(dominantFrequency / herzPerBin); int binBuffer = (int)Math.Round(hzBuffer / herzPerBin);; int dominantBinMin = dominantBin - binBuffer; int dominantBinMax = dominantBin + binBuffer; // freqBin + rowID = binCount - 1; // therefore: rowID = binCount - freqBin - 1; int minRowID = rhzRowCount - dominantBinMax - 1; int maxRowID = rhzRowCount - dominantBinMin - 1; int bottomRow = rhzRowCount - bottomBin - 1; var list = new List <Point>(); // loop through all spectra/columns of the hi-res spectrogram. for (int c = 1; c < rhzColCount - 1; c++) { double maxAmplitude = -double.MaxValue; int idOfRowWithMaxAmplitude = 0; for (int r = minRowID; r <= bottomRow; r++) { if (spg[r, c] > maxAmplitude) { maxAmplitude = spg[r, c]; idOfRowWithMaxAmplitude = r; } } if (idOfRowWithMaxAmplitude < minRowID) { continue; } if (idOfRowWithMaxAmplitude > maxRowID) { continue; } // want a spectral peak. if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c - 1]) { continue; } if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c + 1]) { continue; } // peak should exceed thresold amplitude if (spg[idOfRowWithMaxAmplitude, c] < 3.0) { continue; } // convert row ID to freq bin ID int freqBinID = rhzRowCount - idOfRowWithMaxAmplitude - 1; list.Add(new Point(c, freqBinID)); // we now have a list of potential hits for LimCon. This needs to be filtered. // Console.WriteLine("Col {0}, Bin {1} ", c, freqBinID); } // DEBUG ONLY // ################################ TEMPORARY ################################ // superimpose point on RHZ HiRes spectrogram for debug purposes bool drawOnHiResSpectrogram = true; //string filePath = @"G:\SensorNetworks\Output\Frogs\TestOfHiResIndices-2016July\Test\Towsey.HiResIndices\SpectrogramImages\3mile_creek_dam_-_Herveys_Range_1076_248366_20130305_001700_30_0min.CombinedGreyScale.png"; var fileName = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name); string filePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScale.png"; var debugImage = new FileInfo(filePath); if (!debugImage.Exists) { drawOnHiResSpectrogram = false; } if (drawOnHiResSpectrogram) { // put red dot where max is Bitmap bmp = new Bitmap(filePath); foreach (Point point in list) { bmp.SetPixel(point.X + 70, 1911 - point.Y, Color.Red); } // mark off every tenth frequency bin for (int r = 0; r < 26; r++) { bmp.SetPixel(68, 1911 - (r * 10), Color.Blue); bmp.SetPixel(69, 1911 - (r * 10), Color.Blue); } // mark off upper bound and lower frequency bound bmp.SetPixel(69, 1911 - dominantBinMin, Color.Lime); bmp.SetPixel(69, 1911 - dominantBinMax, Color.Lime); //bmp.SetPixel(69, 1911 - maxRowID, Color.Lime); string opFilePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScaleAnnotated.png"; bmp.Save(opFilePath); } // END DEBUG ################################ TEMPORARY ################################ // now construct the standard decibel spectrogram WITHOUT noise removal, and look for LimConvex // get frame parameters for the analysis double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1); int frameSize = rhzRowCount * 2; int frameStep = frameSize; // this default = zero overlap double frameDurationInSeconds = frameSize / (double)sampleRate; double frameStepInSeconds = frameStep / (double)sampleRate; double framesPerSec = 1 / frameStepInSeconds; //var dspOutput = DSP_Frames.ExtractEnvelopeAndFFTs(recording, frameSize, frameStep); //// Generate deciBel spectrogram //double[,] deciBelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput.amplitudeSpectrogram, dspOutput.WindowPower, sampleRate, epsilon); // i: Init SONOGRAM config var sonoConfig = new SonogramConfig { SourceFName = recording.BaseName, WindowSize = frameSize, WindowOverlap = 0.0, NoiseReductionType = NoiseReductionType.None, }; // init sonogram BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // remove the DC row of the spectrogram sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.Data.GetLength(0) - 1, sonogram.Data.GetLength(1) - 1); //scores.Add(new Plot("Decibels", DataTools.NormaliseMatrixValues(dBArray), ActivityAndCover.DefaultActivityThresholdDb)); //scores.Add(new Plot("Active Frames", DataTools.Bool2Binary(activity.activeFrames), 0.0)); // convert spectral peaks to frequency //var tuple_DecibelPeaks = SpectrogramTools.HistogramOfSpectralPeaks(deciBelSpectrogram); //int[] peaksBins = tuple_DecibelPeaks.Item2; //double[] freqPeaks = new double[peaksBins.Length]; //int binCount = sonogram.Data.GetLength(1); //for (int i = 1; i < peaksBins.Length; i++) freqPeaks[i] = (lowerBinBound + peaksBins[i]) / (double)nyquistBin; //scores.Add(new Plot("Max Frequency", freqPeaks, 0.0)); // location of peaks for spectral images // create new list of LimCon hits in the standard spectrogram. double timeSpanOfFrameInSeconds = frameSize / (double)sampleRate; var newList = new List <int[]>(); int lastFrameID = sonogram.Data.GetLength(0) - 1; int lastBinID = sonogram.Data.GetLength(1) - 1; foreach (Point point in list) { double secondsFromStartOfSegment = (point.X * 0.1) + 0.05; // convert point.Y to center of time-block. int framesFromStartOfSegment = (int)Math.Round(secondsFromStartOfSegment / timeSpanOfFrameInSeconds); // location of max point is uncertain, so search in neighbourhood. // NOTE: sonogram.data matrix is time*freqBin double maxValue = -double.MaxValue; int idOfTMax = framesFromStartOfSegment; int idOfFMax = point.Y; for (int deltaT = -4; deltaT <= 4; deltaT++) { for (int deltaF = -1; deltaF <= 1; deltaF++) { int newT = framesFromStartOfSegment + deltaT; if (newT < 0) { newT = 0; } else if (newT > lastFrameID) { newT = lastFrameID; } double value = sonogram.Data[newT, point.Y + deltaF]; if (value > maxValue) { maxValue = value; idOfTMax = framesFromStartOfSegment + deltaT; idOfFMax = point.Y + deltaF; } } } // newList.Add(new Point(frameSpan, point.Y)); int[] array = new int[2]; array[0] = idOfTMax; array[1] = idOfFMax; newList.Add(array); } // Now obtain more of spectrogram to see if have peaks at two other places characteristic of Limnodynastes convex. // In the D.Stewart CD, there are peaks close to: //1. 1950 Hz //2. 1460 hz //3. 970 hz These are 490 Hz apart. // For Limnodynastes convex, in the JCU recording, there are peaks close to: //1. 1780 Hz //2. 1330 hz //3. 880 hz These are 450 Hz apart. // So strategy is to look for three peaks separated by same amount and in the vicinity of the above, // starting with highest power (the top peak) and working down to lowest power (bottom peak). //We have found top/highest peak - now find the other two. int secondDominantFrequency = 1380; int secondDominantBin = (int)Math.Round(secondDominantFrequency / herzPerBin); int thirdDominantFrequency = 900; int thirdDominantBin = (int)Math.Round(thirdDominantFrequency / herzPerBin); var acousticEvents = new List <AcousticEvent>(); int Tbuffer = 2; // First extract a sub-matrix. foreach (int[] array in newList) { // NOTE: sonogram.data matrix is time*freqBin int Tframe = array[0]; int F1bin = array[1]; double[,] subMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - Tbuffer, 0, Tframe + Tbuffer, F1bin); double F1power = subMatrix[Tbuffer, F1bin]; // convert to vector var spectrum = MatrixTools.GetColumnAverages(subMatrix); // use the following code to get estimate of background noise double[,] powerMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - 3, 10, Tframe + 3, F1bin); double averagePower = (MatrixTools.GetRowAverages(powerMatrix)).Average(); double score = F1power - averagePower; // debug - checking what the spectrum looks like. //for (int i = 0; i < 18; i++) // spectrum[i] = -100.0; //DataTools.writeBarGraph(spectrum); // locate the peaks in lower frequency bands, F2 and F3 bool[] peaks = DataTools.GetPeaks(spectrum); int F2bin = 0; double F2power = -200.0; // dB for (int i = -3; i <= 2; i++) { int bin = F1bin - F1AndF2Gap + i; if ((peaks[bin]) && (F2power < subMatrix[1, bin])) { F2bin = bin; F2power = subMatrix[1, bin]; } } if (F2bin == 0) { continue; } if (F2power == -200.0) { continue; } score += (F2power - averagePower); int F3bin = 0; double F3power = -200.0; for (int i = -5; i <= 2; i++) { int bin = F1bin - F1AndF3Gap + i; if ((peaks[bin]) && (F3power < subMatrix[1, bin])) { F3bin = bin; F3power = subMatrix[1, bin]; } } if (F3bin == 0) { continue; } if (F3power == -200.0) { continue; } score += (F3power - averagePower); score /= 3; // ignore events where SNR < decibel threshold if (score < scoreThreshold) { continue; } // ignore events with wrong power distribution. A good LimnoConvex call has strongest F1 power if ((F3power > F1power) || (F2power > F1power)) { continue; } //freq Bin ID must be converted back to Matrix row ID // freqBin + rowID = binCount - 1; // therefore: rowID = binCount - freqBin - 1; minRowID = rhzRowCount - F1bin - 2; maxRowID = rhzRowCount - F3bin - 1; int F1RowID = rhzRowCount - F1bin - 1; int F2RowID = rhzRowCount - F2bin - 1; int F3RowID = rhzRowCount - F3bin - 1; int maxfreq = dominantFrequency + hzBuffer; int topBin = (int)Math.Round(maxfreq / herzPerBin); int frameCount = 4; double duration = frameCount * frameStepInSeconds; double startTimeWrtSegment = (Tframe - 2) * frameStepInSeconds; // Got to here so start initialising an acoustic event var ae = new AcousticEvent(segmentStartOffset, startTimeWrtSegment, duration, minimumFrequency, maxfreq); ae.SetTimeAndFreqScales(framesPerSec, herzPerBin); //var ae = new AcousticEvent(oblong, recording.Nyquist, binCount, frameDurationInSeconds, frameStepInSeconds, frameCount); //ae.StartOffset = TimeSpan.FromSeconds(Tframe * frameStepInSeconds); var pointF1 = new Point(2, topBin - F1bin); var pointF2 = new Point(2, topBin - F2bin); var pointF3 = new Point(2, topBin - F3bin); ae.Points = new List <Point>(); ae.Points.Add(pointF1); ae.Points.Add(pointF2); ae.Points.Add(pointF3); //tried using HitElements but did not do what I wanted later on. //ae.HitElements = new HashSet<Point>(); //ae.HitElements = new SortedSet<Point>(); //ae.HitElements.Add(pointF1); //ae.HitElements.Add(pointF2); //ae.HitElements.Add(pointF3); ae.Score = score; //ae.MinFreq = Math.Round((topBin - F3bin - 5) * herzPerBin); //ae.MaxFreq = Math.Round(topBin * herzPerBin); acousticEvents.Add(ae); } // now add in extra common info to the acoustic events acousticEvents.ForEach(ae => { ae.SpeciesName = configDict[AnalysisKeys.SpeciesName]; ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; ae.SegmentDurationSeconds = recording.Duration.TotalSeconds; ae.Name = abbreviatedName; ae.BorderColour = Color.Red; ae.FileName = recording.BaseName; }); double[] scores = new double[rhzColCount]; // predefinition of score array double nomalisationConstant = scoreThreshold * 4; // four times the score threshold double compressionFactor = rhzColCount / (double)sonogram.Data.GetLength(0); foreach (AcousticEvent ae in acousticEvents) { ae.ScoreNormalised = ae.Score / nomalisationConstant; if (ae.ScoreNormalised > 1.0) { ae.ScoreNormalised = 1.0; } int frameID = (int)Math.Round(ae.EventStartSeconds / frameDurationInSeconds); int hiresFrameID = (int)Math.Floor(frameID * compressionFactor); scores[hiresFrameID] = ae.ScoreNormalised; } var plot = new Plot(AnalysisName, scores, scoreThreshold); // DEBUG ONLY ################################ TEMPORARY ################################ // Draw a standard spectrogram and mark of hites etc. bool createStandardDebugSpectrogram = true; var imageDir = new DirectoryInfo(outputDir.FullName + @"\SpectrogramImages"); if (!imageDir.Exists) { imageDir.Create(); } if (createStandardDebugSpectrogram) { var fileName2 = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name); string filePath2 = Path.Combine(imageDir.FullName, fileName + ".Spectrogram.png"); Bitmap sonoBmp = (Bitmap)sonogram.GetImage(); int height = sonoBmp.Height; foreach (AcousticEvent ae in acousticEvents) { ae.DrawEvent(sonoBmp); //g.DrawRectangle(pen, ob.ColumnLeft, ob.RowTop, ob.ColWidth-1, ob.RowWidth); //ae.DrawPoint(sonoBmp, ae.HitElements.[0], Color.OrangeRed); //ae.DrawPoint(sonoBmp, ae.HitElements[1], Color.Yellow); //ae.DrawPoint(sonoBmp, ae.HitElements[2], Color.Green); ae.DrawPoint(sonoBmp, ae.Points[0], Color.OrangeRed); ae.DrawPoint(sonoBmp, ae.Points[1], Color.Yellow); ae.DrawPoint(sonoBmp, ae.Points[2], Color.LimeGreen); } // draw the original hits on the standard sonogram foreach (int[] array in newList) { sonoBmp.SetPixel(array[0], height - array[1], Color.Cyan); } // mark off every tenth frequency bin on the standard sonogram for (int r = 0; r < 20; r++) { sonoBmp.SetPixel(0, height - (r * 10) - 1, Color.Blue); sonoBmp.SetPixel(1, height - (r * 10) - 1, Color.Blue); } // mark off upper bound and lower frequency bound sonoBmp.SetPixel(0, height - dominantBinMin, Color.Lime); sonoBmp.SetPixel(0, height - dominantBinMax, Color.Lime); sonoBmp.Save(filePath2); } // END DEBUG ################################ TEMPORARY ################################ return(new LimnodynastesConvexResults { Sonogram = sonogram, Hits = null, Plot = plot, Events = acousticEvents, RecordingDuration = recording.Duration, }); } // Analysis()
/// <summary> /// Calculate summary statistics for supplied temporal and spectral targets. /// </summary> /// <remarks> /// The acoustic statistics calculated in this method are based on methods outlined in /// "Acoustic classification of multiple simultaneous bird species: A multi-instance multi-label approach", /// by Forrest Briggs, Balaji Lakshminarayanan, Lawrence Neal, Xiaoli Z.Fern, Raviv Raich, Sarah J.K.Hadley, Adam S. Hadley, Matthew G. Betts, et al. /// The Journal of the Acoustical Society of America v131, pp4640 (2012); doi: http://dx.doi.org/10.1121/1.4707424 /// .. /// The Briggs feature are calculated from the column (freq bin) and row (frame) sums of the extracted spectrogram. /// 1. Gini Index for frame and bin sums. A measure of dispersion. Problem with gini is that its value is dependent on the row or column count. /// We use entropy instead because value not dependent on row or column count because it is normalized. /// For the following meausres of k-central moments, the freq and time values are normalized in 0,1 to width of the event. /// 2. freq-mean /// 3. freq-variance /// 4. freq-skew and kurtosis /// 5. time-mean /// 6. time-variance /// 7. time-skew and kurtosis /// 8. freq-max (normalized) /// 9. time-max (normalized) /// 10. Briggs et al also calculate a 16 value histogram of gradients for each event mask. We do not do that here although we could. /// ... /// NOTE 1: There are differences between our method of noise reduction and Briggs. Briggs does not convert to decibels /// and instead works with power values. He obtains a noise profile from the 20% of frames having the lowest energy sum. /// NOTE 2: To NormaliseMatrixValues for noise, they divide the actual energy by the noise value. This is equivalent to subtraction when working in decibels. /// There are advantages and disadvantages to Briggs method versus ours. In our case, we hve to convert decibel values back to /// energy values when calculating the statistics for the extracted acoustic event. /// NOTE 3: We do not calculate the higher central moments of the time/frequency profiles, i.e. skew and kurtosis. /// Ony mean and standard deviation. /// .. /// NOTE 4: This method assumes that the passed event occurs totally within the passed recording, /// AND that the passed recording is of sufficient duration to obtain reliable BGN noise profile /// BUT not so long as to cause memory constipation. /// </remarks> /// <param name="recording">as type AudioRecording which contains the event</param> /// <param name="temporalTarget">Both start and end bounds - relative to the supplied recording</param> /// <param name="spectralTarget">both bottom and top bounds in Hertz</param> /// <param name="config">parameters that determine the outcome of the analysis</param> /// <param name="segmentStartOffset">How long since the start of the recording this event occurred</param> /// <returns>an instance of EventStatistics</returns> public static EventStatistics AnalyzeAudioEvent( AudioRecording recording, Range <TimeSpan> temporalTarget, Range <double> spectralTarget, EventStatisticsConfiguration config, TimeSpan segmentStartOffset) { var stats = new EventStatistics { EventStartSeconds = temporalTarget.Minimum.TotalSeconds, EventEndSeconds = temporalTarget.Maximum.TotalSeconds, LowFrequencyHertz = spectralTarget.Minimum, HighFrequencyHertz = spectralTarget.Maximum, SegmentDurationSeconds = recording.Duration.TotalSeconds, SegmentStartSeconds = segmentStartOffset.TotalSeconds, }; // temporal target is supplied relative to recording, but not the supplied audio segment // shift coordinates relative to segment var localTemporalTarget = temporalTarget.Shift(-segmentStartOffset); if (!recording .Duration .AsRangeFromZero(Topology.Inclusive) .Contains(localTemporalTarget)) { stats.Error = true; stats.ErrorMessage = $"Audio not long enough ({recording.Duration}) to analyze target ({localTemporalTarget})"; return(stats); } // convert recording to spectrogram int sampleRate = recording.SampleRate; double epsilon = recording.Epsilon; // extract the spectrogram var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recording, config.FrameSize, config.FrameStep); double hertzBinWidth = dspOutput1.FreqBinWidth; var stepDurationInSeconds = config.FrameStep / (double)sampleRate; var startFrame = (int)Math.Ceiling(localTemporalTarget.Minimum.TotalSeconds / stepDurationInSeconds); // subtract 1 frame because want to end before start of end point. var endFrame = (int)Math.Floor(localTemporalTarget.Maximum.TotalSeconds / stepDurationInSeconds) - 1; var bottomBin = (int)Math.Floor(spectralTarget.Minimum / hertzBinWidth); var topBin = (int)Math.Ceiling(spectralTarget.Maximum / hertzBinWidth); // Events can have their high value set to the nyquist. // Since the submatrix call below uses an inclusive upper bound an index out of bounds exception occurs in // these cases. So we just ask for the bin below. if (topBin >= config.FrameSize / 2) { topBin = (config.FrameSize / 2) - 1; } // Convert amplitude spectrogram to deciBels and calculate the dB background noise profile double[,] decibelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput1.AmplitudeSpectrogram, dspOutput1.WindowPower, sampleRate, epsilon); double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram); decibelSpectrogram = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram, spectralDecibelBgn); decibelSpectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram, nhThreshold: 2.0); // extract the required acoustic event var eventMatrix = MatrixTools.Submatrix(decibelSpectrogram, startFrame, bottomBin, endFrame, topBin); // Get the SNR of the event. This is just the max value in the matrix because noise reduced MatrixTools.MinMax(eventMatrix, out _, out double max); stats.SnrDecibels = max; // Now need to convert event matrix back to energy values before calculating other statistics eventMatrix = MatrixTools.Decibels2Power(eventMatrix); var columnAverages = MatrixTools.GetColumnAverages(eventMatrix); var rowAverages = MatrixTools.GetRowAverages(eventMatrix); // calculate the mean and temporal standard deviation in decibels NormalDist.AverageAndSD(rowAverages, out double mean, out double stddev); stats.MeanDecibels = 10 * Math.Log10(mean); stats.TemporalStdDevDecibels = 10 * Math.Log10(stddev); // calculate the frequency standard deviation in decibels NormalDist.AverageAndSD(columnAverages, out mean, out stddev); stats.FreqBinStdDevDecibels = 10 * Math.Log10(stddev); // calculate relative location of the temporal maximum int maxRowId = DataTools.GetMaxIndex(rowAverages); stats.TemporalMaxRelative = maxRowId / (double)rowAverages.Length; // calculate the entropy dispersion/concentration indices stats.TemporalEnergyDistribution = 1 - DataTools.EntropyNormalised(rowAverages); stats.SpectralEnergyDistribution = 1 - DataTools.EntropyNormalised(columnAverages); // calculate the spectral centroid and the dominant frequency double binCentroid = CalculateSpectralCentroid(columnAverages); stats.SpectralCentroid = (int)Math.Round(hertzBinWidth * binCentroid) + (int)spectralTarget.Minimum; int maxColumnId = DataTools.GetMaxIndex(columnAverages); stats.DominantFrequency = (int)Math.Round(hertzBinWidth * maxColumnId) + (int)spectralTarget.Minimum; // remainder of this method is to produce debugging images. Can comment out when not debugging. /* * var normalisedIndex = DataTools.NormaliseMatrixValues(columnAverages); * var image4 = GraphsAndCharts.DrawGraph("columnSums", normalisedIndex, 100); * string path4 = @"C:\SensorNetworks\Output\Sonograms\UnitTestSonograms\columnSums.png"; * image4.Save(path4); * normalisedIndex = DataTools.NormaliseMatrixValues(rowAverages); * image4 = GraphsAndCharts.DrawGraph("rowSums", normalisedIndex, 100); * path4 = @"C:\SensorNetworks\Output\Sonograms\UnitTestSonograms\rowSums.png"; * image4.Save(path4); */ return(stats); }
//public const string key_COUNT = "count"; public static Tuple <double[, ], double[, ], double[, ], double[]> DetectBarsUsingXcorrelation(double[,] m, int rowStep, int rowWidth, int colStep, int colWidth, double intensityThreshold, int zeroBinCount) { bool doNoiseremoval = true; //intensityThreshold = 0.3; int rowCount = m.GetLength(0); int colCount = m.GetLength(1); int numberOfColSteps = colCount / colStep; int numberOfRowSteps = rowCount / rowStep; var intensityMatrix = new double[numberOfRowSteps, numberOfColSteps]; var periodicityMatrix = new double[numberOfRowSteps, numberOfColSteps]; var hitsMatrix = new double[rowCount, colCount]; double[] array2return = null; for (int b = 0; b < numberOfColSteps; b++) { int minCol = b * colStep; int maxCol = minCol + colWidth - 1; double[,] subMatrix = MatrixTools.Submatrix(m, 0, minCol, rowCount - 1, maxCol); double[] amplitudeArray = MatrixTools.GetRowAverages(subMatrix); if (doNoiseremoval) { double StandardDeviationCount = 0.1; // number of noise SDs to calculate noise threshold - determines severity of noise reduction SNR.BackgroundNoise bgn = SNR.SubtractBackgroundNoiseFromSignal(amplitudeArray, StandardDeviationCount); amplitudeArray = bgn.NoiseReducedSignal; } //double noiseThreshold = 0.005; //for (int i = 1; i < amplitudeArray.Length - 1; i++) //{ // if ((amplitudeArray[i - 1] < noiseThreshold) && (amplitudeArray[i + 1] < noiseThreshold)) amplitudeArray[i] = 0.0; //} //DataTools.writeBarGraph(amplitudeArray); if (b == 2) { array2return = amplitudeArray; //returned for debugging purposes only } //ii: DETECT HARMONICS var results = AutoAndCrossCorrelation.DetectPeriodicityInLongArray(amplitudeArray, rowStep, rowWidth, zeroBinCount); double[] intensity = results.Item1; //an array of periodicity scores double[] periodicity = results.Item2; //transfer periodicity info to a matrices. for (int rs = 0; rs < numberOfRowSteps; rs++) { intensityMatrix[rs, b] = intensity[rs]; periodicityMatrix[rs, b] = periodicity[rs]; //mark up the hits matrix //double relativePeriod = periodicity[rs] / rowWidth / 2; if (intensity[rs] > intensityThreshold) { int minRow = rs * rowStep; int maxRow = minRow + rowStep - 1; for (int r = minRow; r < maxRow; r++) { for (int c = minCol; c < maxCol; c++) { //hitsMatrix[r, c] = relativePeriod; hitsMatrix[r, c] = periodicity[rs]; } } } // if() } // for loop over numberOfRowSteps } // for loop over numberOfColSteps return(Tuple.Create(intensityMatrix, periodicityMatrix, hitsMatrix, array2return)); }
} //DetectBarsInTheRowsOfaMatrix() /// A METHOD TO DETECT HARMONICS IN THE ROWS of the passed portion of a sonogram. /// This method assume the matrix is derived from a spectrogram rotated so that the matrix rows are spectral columns of sonogram. /// Was first developed for crow calls. /// First looks for a decibel profile that matches the passed call duration and decibel loudness /// Then samples the centre portion for the correct harmonic period. /// </summary> /// <param name="m"></param> /// <param name="amplitudeThreshold"></param> /// <returns></returns> public static Tuple <double[], double[], double[]> DetectHarmonicsInSonogramMatrix(double[,] m, double dBThreshold, int callSpan) { int zeroBinCount = 3; //to remove low freq content which dominates the spectrum int halfspan = callSpan / 2; double[] dBArray = MatrixTools.GetRowAverages(m); dBArray = DataTools.filterMovingAverage(dBArray, 3); bool doNoiseRemoval = true; if (doNoiseRemoval) { double StandardDeviationCount = 0.1; // number of noise SDs to calculate noise threshold - determines severity of noise reduction SNR.BackgroundNoise bgn = SNR.SubtractBackgroundNoiseFromSignal(dBArray, StandardDeviationCount); dBArray = bgn.NoiseReducedSignal; } bool[] peaks = DataTools.GetPeaks(dBArray); int rowCount = m.GetLength(0); int colCount = m.GetLength(1); var intensity = new double[rowCount]; //an array of period intensity var periodicity = new double[rowCount]; //an array of the periodicity values for (int r = halfspan; r < rowCount - halfspan; r++) { //APPLY A FILTER: must satisfy the following conditions for a call. if (!peaks[r]) { continue; } if (dBArray[r] < dBThreshold) { continue; } double lowerDiff = dBArray[r] - dBArray[r - halfspan]; double upperDiff = dBArray[r] - dBArray[r + halfspan]; if (lowerDiff < dBThreshold || upperDiff < dBThreshold) { continue; } double[] prevRow = DataTools.DiffFromMean(MatrixTools.GetRow(m, r - 1)); double[] thisRow = DataTools.DiffFromMean(MatrixTools.GetRow(m, r)); var spectrum = AutoAndCrossCorrelation.CrossCorr(prevRow, thisRow); for (int s = 0; s < zeroBinCount; s++) { spectrum[s] = 0.0; //in real data these bins are dominant and hide other frequency content } spectrum = DataTools.NormaliseArea(spectrum); int maxId = DataTools.GetMaxIndex(spectrum); double intensityValue = spectrum[maxId]; intensity[r] = intensityValue; double period = 0.0; if (maxId != 0) { period = 2 * colCount / (double)maxId; } periodicity[r] = period; prevRow = thisRow; } // rows return(Tuple.Create(dBArray, intensity, periodicity)); } //DetectHarmonicsInSonogramMatrix()
public static double[,] GetFrequencyByOscillationsMatrix(double[,] spectrogram, double sensitivity, int sampleLength, string algorithmName) { int frameCount = spectrogram.GetLength(0); int freqBinCount = spectrogram.GetLength(1); var freqByOscMatrix = new double[sampleLength / 2, freqBinCount]; // over all frequency bins for (int bin = 0; bin < freqBinCount; bin++) { double[,] subM; // get average of three bins if (bin == 0) { subM = MatrixTools.Submatrix(spectrogram, 0, 0, frameCount - 1, 2); } else // get average of three bins if (bin == freqBinCount - 1) { subM = MatrixTools.Submatrix(spectrogram, 0, bin - 2, frameCount - 1, bin); } else { // get average of three bins subM = MatrixTools.Submatrix(spectrogram, 0, bin - 1, frameCount - 1, bin + 1); } var freqBin = MatrixTools.GetRowAverages(subM); // could alternatively take single bins but averaging three bins seems to work well //var freqBin = MatrixTools.GetColumn(spectrogram, bin); // vector to store the oscillations vector derived from one frequency bin. double[] oscillationsSpectrum = null; var xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength); // Use the Autocorrelation - FFT option. // This option appears to work best for use as a spectral index to detect oscillations if (algorithmName.Equals("autocorr-fft")) { oscillationsSpectrum = GetOscillationArrayUsingFft(xCorrByTimeMatrix, sensitivity); } // Use the Autocorrelation - SVD - FFT option. if (algorithmName.Equals("autocorr-svd-fft")) { oscillationsSpectrum = GetOscillationArrayUsingSvdAndFft(xCorrByTimeMatrix, sensitivity, bin); } // Use the Wavelet Transform if (algorithmName.Equals("Autocorr-WPD")) { oscillationsSpectrum = GetOscillationArrayUsingWpd(xCorrByTimeMatrix, sensitivity, bin); //WaveletTransformContinuous cwt = new WaveletTransformContinuous(freqBin, maxScale); //double[,] cwtMatrix = cwt.GetScaleTimeMatrix(); //oscillationsSpectrum = GetOscillationArrayUsingCWT(cwtMatrix, sensitivity, bin); //double[] dynamicRanges = GetVectorOfDynamicRanges(freqBin, sampleLength); } // transfer final oscillation vector for single frequency bin to the Oscillations by frequency matrix. MatrixTools.SetColumn(freqByOscMatrix, bin, oscillationsSpectrum); } // foreach frequency bin return(freqByOscMatrix); }
public static double[,] GetFrequencyByOscillationsMatrix(double[,] spectrogram, double sensitivity, int sampleLength, string algorithmName) { int frameCount = spectrogram.GetLength(0); int freqBinCount = spectrogram.GetLength(1); double[,] freqByOscMatrix = new double[sampleLength / 2, freqBinCount]; // over all frequency bins for (int bin = 0; bin < freqBinCount; bin++) { //bin = 50; // for debugging //Console.WriteLine("Bin = {0}", bin); double[,] subM; // get average of three bins if (bin == 0) { subM = MatrixTools.Submatrix(spectrogram, 0, 0, frameCount - 1, 2); } else // get average of three bins if (bin == freqBinCount - 1) { subM = MatrixTools.Submatrix(spectrogram, 0, bin - 2, frameCount - 1, bin); } else { // get average of three bins subM = MatrixTools.Submatrix(spectrogram, 0, bin - 1, frameCount - 1, bin + 1); } var freqBin = MatrixTools.GetRowAverages(subM); // vector to store the oscilations vector derived from one frequency bin. double[] oscillationsSpectrum = null; // Use the Autocorrelation - SVD - FFT option. if (algorithmName.Equals("autocorr-svd-fft")) { double[,] xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength); //xcorCount += xCorrByTimeMatrix.GetLength(1); oscillationsSpectrum = GetOscillationArrayUsingSvdAndFft(xCorrByTimeMatrix, sensitivity, bin); } // Use the Autocorrelation - FFT option. if (algorithmName.Equals("autocorr-fft")) { double[,] xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength); oscillationsSpectrum = GetOscillationArrayUsingFft(xCorrByTimeMatrix, sensitivity, bin); } // Use the Wavelet Transform if (algorithmName.Equals("Autocorr-WPD")) { double[,] xCorrByTimeMatrix = GetXcorrByTimeMatrix(freqBin, sampleLength); oscillationsSpectrum = GetOscillationArrayUsingWpd(xCorrByTimeMatrix, sensitivity, bin); //WaveletTransformContinuous cwt = new WaveletTransformContinuous(freqBin, maxScale); //double[,] cwtMatrix = cwt.GetScaleTimeMatrix(); //oscillationsSpectrum = GetOscillationArrayUsingCWT(cwtMatrix, sensitivity, bin); //double[] dynamicRanges = GetVectorOfDynamicRanges(freqBin, sampleLength); } // transfer final oscillation vector to the Oscillations by frequency matrix. MatrixTools.SetColumn(freqByOscMatrix, bin, oscillationsSpectrum); } // feareach frequency bin return(freqByOscMatrix); }