public static Tuple <BaseSonogram, AcousticEvent, double[, ], double[], double[, ]> Execute_Extraction( AudioRecording recording, double eventStart, double eventEnd, int minHz, int maxHz, double frameOverlap, double backgroundThreshold, TimeSpan segmentStartOffset) { //ii: MAKE SONOGRAM SonogramConfig sonoConfig = new SonogramConfig(); //default values config sonoConfig.SourceFName = recording.BaseName; //sonoConfig.WindowSize = windowSize; sonoConfig.WindowOverlap = frameOverlap; BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); Log.WriteLine("Frames: Size={0}, Count={1}, Duration={2:f1}ms, Overlap={5:f2}%, Offset={3:f1}ms, Frames/s={4:f1}", sonogram.Configuration.WindowSize, sonogram.FrameCount, (sonogram.FrameDuration * 1000), (sonogram.FrameStep * 1000), sonogram.FramesPerSecond, frameOverlap); int binCount = (int)(maxHz / sonogram.FBinWidth) - (int)(minHz / sonogram.FBinWidth) + 1; Log.WriteIfVerbose("Freq band: {0} Hz - {1} Hz. (Freq bin count = {2})", minHz, maxHz, binCount); //calculate the modal noise profile double SD_COUNT = 0.0; // number of noise standard deviations used to calculate noise threshold NoiseProfile profile = NoiseProfile.CalculateModalNoiseProfile(sonogram.Data, SD_COUNT); //calculate modal noise profile double[] modalNoise = DataTools.filterMovingAverage(profile.NoiseMode, 7); //smooth the noise profile //extract modal noise values of the required event double[] noiseSubband = SpectrogramTools.ExtractModalNoiseSubband(modalNoise, minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth); //extract data values of the required event double[,] target = SpectrogramTools.ExtractEvent(sonogram.Data, eventStart, eventEnd, sonogram.FrameStep, minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth); // create acoustic event with defined boundaries AcousticEvent ae = new AcousticEvent(segmentStartOffset, eventStart, eventEnd - eventStart, minHz, maxHz); ae.SetTimeAndFreqScales(sonogram.FramesPerSecond, sonogram.FBinWidth); //truncate noise sonogram.Data = SNR.TruncateBgNoiseFromSpectrogram(sonogram.Data, modalNoise); sonogram.Data = SNR.RemoveNeighbourhoodBackgroundNoise(sonogram.Data, backgroundThreshold); double[,] targetMinusNoise = SpectrogramTools.ExtractEvent(sonogram.Data, eventStart, eventEnd, sonogram.FrameStep, minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth); return(Tuple.Create(sonogram, ae, target, noiseSubband, targetMinusNoise)); }
public static double[,] GetDecibelSpectrogramNoiseReduced(AudioRecording recording, int frameSize) { int frameStep = frameSize; // get decibel spectrogram var results = DSP_Frames.ExtractEnvelopeAndAmplSpectrogram(recording.WavReader.Samples, recording.SampleRate, recording.Epsilon, frameSize, frameStep); var spectrogram = MFCCStuff.DecibelSpectra(results.AmplitudeSpectrogram, results.WindowPower, recording.SampleRate, recording.Epsilon); // remove background noise from spectrogram double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(spectrogram); spectrogram = SNR.TruncateBgNoiseFromSpectrogram(spectrogram, spectralDecibelBgn); spectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(spectrogram, nhThreshold: 3.0); return(spectrogram); }
/// <summary> /// This method produces four spectrograms using four different values of neighbour hood decibel threshold. /// It can be used for test purposes. /// </summary> /// <param name="deciBelSpectrogram">the noisy decibel spectrogram</param> /// <param name="xAxisInterval">x-axis tic interval</param> /// <param name="stepDuration">the x-axis times scale</param> /// <param name="nyquist">max freq value</param> /// <param name="hzInterval">y-axis frequency scale</param> /// <returns>Image containing four sepctrograms</returns> public static Image ModalNoiseRemovalAndGetSonograms( double[,] deciBelSpectrogram, TimeSpan xAxisInterval, TimeSpan stepDuration, int nyquist, int hzInterval) { // The number of SDs above the mean for noise removal. // Set sdCount = -0.5 becuase when sdCount >= zero, noies removal is a bit severe for environmental recordings. var sdCount = -0.5; var nrt = NoiseReductionType.Modal; var tuple = SNR.NoiseReduce(deciBelSpectrogram, nrt, sdCount); var noiseReducedSpectrogram1 = tuple.Item1; var title = "title1"; var image1 = DrawSonogram(noiseReducedSpectrogram1, xAxisInterval, stepDuration, nyquist, hzInterval, title); double dBThreshold = 0.0; // SPECTRAL dB THRESHOLD for smoothing background double[,] noiseReducedSpectrogram2 = SNR.RemoveNeighbourhoodBackgroundNoise(noiseReducedSpectrogram1, dBThreshold); title = "title2"; var image2 = DrawSonogram(noiseReducedSpectrogram2, xAxisInterval, stepDuration, nyquist, hzInterval, title); // SPECTRAL dB THRESHOLD for smoothing background dBThreshold = 3.0; noiseReducedSpectrogram2 = SNR.RemoveNeighbourhoodBackgroundNoise(noiseReducedSpectrogram1, dBThreshold); title = "title3"; var image3 = DrawSonogram(noiseReducedSpectrogram2, xAxisInterval, stepDuration, nyquist, hzInterval, title); // SPECTRAL dB THRESHOLD for smoothing background dBThreshold = 10.0; noiseReducedSpectrogram2 = SNR.RemoveNeighbourhoodBackgroundNoise(noiseReducedSpectrogram1, dBThreshold); title = "title4"; var image4 = DrawSonogram(noiseReducedSpectrogram2, xAxisInterval, stepDuration, nyquist, hzInterval, title); var array = new Image[4]; array[0] = image1; array[1] = image2; array[2] = image3; array[3] = image4; var combinedImage = ImageTools.CombineImagesVertically(array); return(combinedImage); }
} // LocalPeaks() /// <summary> /// CALCULATEs SPECTRAL PEAK TRACKS: spectralIndices.SPT, RHZ, RVT, RPS, RNG /// This method is only called from IndexCalulate.analysis() when the IndexCalculation Duration is less than 10 seconds, /// because need to recalculate background noise etc. /// Otherwise the constructor of this class is called: sptInfo = new SpectralPeakTracks(decibelSpectrogram, peakThreshold); /// NOTE: We require a noise reduced decibel spectrogram /// FreqBinWidth can be accessed, if required, through dspOutput1.FreqBinWidth. /// </summary> public static SpectralPeakTracks CalculateSpectralPeakTracks(AudioRecording recording, int sampleStart, int sampleEnd, int frameSize, bool octaveScale, double peakThreshold) { double epsilon = recording.Epsilon; int sampleRate = recording.WavReader.SampleRate; int bufferFrameCount = 2; // 2 because must allow for edge effects when using 5x5 grid to find ridges. int ridgeBuffer = frameSize * bufferFrameCount; var ridgeRecording = AudioRecording.GetRecordingSubsegment(recording, sampleStart, sampleEnd, ridgeBuffer); int frameStep = frameSize; var dspOutput = DSP_Frames.ExtractEnvelopeAndFfts(ridgeRecording, frameSize, frameStep); // Generate the ridge SUBSEGMENT deciBel spectrogram from the SUBSEGMENT amplitude spectrogram // i: generate the SUBSEGMENT deciBel spectrogram from the SUBSEGMENT amplitude spectrogram double[,] decibelSpectrogram; if (octaveScale) { var freqScale = new FrequencyScale(FreqScaleType.Linear125Octaves7Tones28Nyquist32000); decibelSpectrogram = OctaveFreqScale.DecibelSpectra(dspOutput.AmplitudeSpectrogram, dspOutput.WindowPower, sampleRate, epsilon, freqScale); } else { decibelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput.AmplitudeSpectrogram, dspOutput.WindowPower, sampleRate, epsilon); } // calculate the noise profile var spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram); decibelSpectrogram = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram, spectralDecibelBgn); double nhDecibelThreshold = 2.0; // SPECTRAL dB THRESHOLD for smoothing background decibelSpectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram, nhDecibelThreshold); // thresholds in decibels // double frameStepDuration = frameStep / (double)sampleRate; // fraction of a second // TimeSpan frameStepTimeSpan = TimeSpan.FromTicks((long)(frameStepDuration * TimeSpan.TicksPerSecond)); var sptInfo = new SpectralPeakTracks(decibelSpectrogram, peakThreshold); return(sptInfo); }
//////public static IndexCalculateResult Analysis( public static SpectralIndexValuesForContentDescription Analysis( AudioRecording recording, TimeSpan segmentOffsetTimeSpan, int sampleRateOfOriginalAudioFile, bool returnSonogramInfo = false) { // returnSonogramInfo = true; // if debugging double epsilon = recording.Epsilon; int sampleRate = recording.WavReader.SampleRate; //var segmentDuration = TimeSpan.FromSeconds(recording.WavReader.Time.TotalSeconds); var indexCalculationDuration = TimeSpan.FromSeconds(ContentSignatures.IndexCalculationDurationInSeconds); // Get FRAME parameters for the calculation of Acoustic Indices int frameSize = ContentSignatures.FrameSize; int frameStep = frameSize; // that is, windowOverlap = zero double frameStepDuration = frameStep / (double)sampleRate; // fraction of a second var frameStepTimeSpan = TimeSpan.FromTicks((long)(frameStepDuration * TimeSpan.TicksPerSecond)); // INITIALISE a RESULTS STRUCTURE TO return // initialize a result object in which to store SummaryIndexValues and SpectralIndexValues etc. var config = new IndexCalculateConfig(); // sets some default values int freqBinCount = frameSize / 2; var indexProperties = GetIndexProperties(); ////////var result = new IndexCalculateResult(freqBinCount, indexProperties, indexCalculationDuration, segmentOffsetTimeSpan, config); var spectralIndices = new SpectralIndexValuesForContentDescription(); ///////result.SummaryIndexValues = null; ///////SpectralIndexValues spectralIndices = result.SpectralIndexValues; // set up default spectrogram to return ///////result.Sg = returnSonogramInfo ? GetSonogram(recording, windowSize: 1024) : null; ///////result.Hits = null; ///////result.TrackScores = new List<Plot>(); // ################################## FINISHED SET-UP // ################################## NOW GET THE AMPLITUDE SPECTROGRAM // EXTRACT ENVELOPE and SPECTROGRAM FROM RECORDING SEGMENT // Note that the amplitude spectrogram has had the DC bin removed. i.e. has only 256 columns. var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recording, frameSize, frameStep); var amplitudeSpectrogram = dspOutput1.AmplitudeSpectrogram; // (B) ################################## EXTRACT OSC SPECTRAL INDEX DIRECTLY FROM THE RECORDING ################################## // Get the oscillation spectral index OSC separately from signal because need a different frame size etc. var sampleLength = Oscillations2014.DefaultSampleLength; var frameLength = Oscillations2014.DefaultFrameLength; var sensitivity = Oscillations2014.DefaultSensitivityThreshold; var spectralIndexShort = Oscillations2014.GetSpectralIndex_Osc(recording, frameLength, sampleLength, sensitivity); // double length of the vector because want to work with 256 element vector for spectrogram purposes spectralIndices.OSC = DataTools.VectorDoubleLengthByAverageInterpolation(spectralIndexShort); // (C) ################################## EXTRACT SPECTRAL INDICES FROM THE AMPLITUDE SPECTROGRAM ################################## // IFF there has been UP-SAMPLING, calculate bin of the original audio nyquist. this will be less than SR/2. // original sample rate can be anything 11.0-44.1 kHz. int originalNyquist = sampleRateOfOriginalAudioFile / 2; // if up-sampling has been done if (dspOutput1.NyquistFreq > originalNyquist) { dspOutput1.NyquistFreq = originalNyquist; dspOutput1.NyquistBin = (int)Math.Floor(originalNyquist / dspOutput1.FreqBinWidth); // note that bin width does not change } // ii: CALCULATE THE ACOUSTIC COMPLEXITY INDEX spectralIndices.ACI = AcousticComplexityIndex.CalculateAci(amplitudeSpectrogram); // iii: CALCULATE the H(t) or Temporal ENTROPY Spectrum and then reverse the values i.e. calculate 1-Ht for energy concentration double[] temporalEntropySpectrum = AcousticEntropy.CalculateTemporalEntropySpectrum(amplitudeSpectrogram); for (int i = 0; i < temporalEntropySpectrum.Length; i++) { temporalEntropySpectrum[i] = 1 - temporalEntropySpectrum[i]; } spectralIndices.ENT = temporalEntropySpectrum; // (C) ################################## EXTRACT SPECTRAL INDICES FROM THE DECIBEL SPECTROGRAM ################################## // i: Convert amplitude spectrogram to decibels and calculate the dB background noise profile double[,] decibelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput1.AmplitudeSpectrogram, dspOutput1.WindowPower, sampleRate, epsilon); double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram); spectralIndices.BGN = spectralDecibelBgn; // ii: Calculate the noise reduced decibel spectrogram derived from segment recording. // REUSE the var decibelSpectrogram but this time using dspOutput1. decibelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput1.AmplitudeSpectrogram, dspOutput1.WindowPower, sampleRate, epsilon); decibelSpectrogram = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram, spectralDecibelBgn); decibelSpectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram, nhThreshold: 2.0); // iii: CALCULATE noise reduced AVERAGE DECIBEL SPECTRUM spectralIndices.PMN = SpectrogramTools.CalculateAvgDecibelSpectrumFromDecibelSpectrogram(decibelSpectrogram); // ###################################################################################################################################################### // iv: CALCULATE SPECTRAL COVER. NOTE: at this point, decibelSpectrogram is noise reduced. All values >= 0.0 // FreqBinWidth can be accessed, if required, through dspOutput1.FreqBinWidth // dB THRESHOLD for calculating spectral coverage double dBThreshold = ActivityAndCover.DefaultActivityThresholdDb; // Calculate lower and upper boundary bin ids. // Boundary between low & mid frequency bands is to avoid low freq bins containing anthropogenic noise. These biased index values away from bio-phony. int midFreqBound = config.MidFreqBound; int lowFreqBound = config.LowFreqBound; int lowerBinBound = (int)Math.Ceiling(lowFreqBound / dspOutput1.FreqBinWidth); int middleBinBound = (int)Math.Ceiling(midFreqBound / dspOutput1.FreqBinWidth); var spActivity = ActivityAndCover.CalculateSpectralEvents(decibelSpectrogram, dBThreshold, frameStepTimeSpan, lowerBinBound, middleBinBound); //spectralIndices.CVR = spActivity.CoverSpectrum; spectralIndices.EVN = spActivity.EventSpectrum; ///////result.TrackScores = null; ///////return result; return(spectralIndices); } // end calculation of Six Spectral Indices
/// <summary> /// Calculate summary statistics for supplied temporal and spectral targets. /// </summary> /// <remarks> /// The acoustic statistics calculated in this method are based on methods outlined in /// "Acoustic classification of multiple simultaneous bird species: A multi-instance multi-label approach", /// by Forrest Briggs, Balaji Lakshminarayanan, Lawrence Neal, Xiaoli Z.Fern, Raviv Raich, Sarah J.K.Hadley, Adam S. Hadley, Matthew G. Betts, et al. /// The Journal of the Acoustical Society of America v131, pp4640 (2012); doi: http://dx.doi.org/10.1121/1.4707424 /// .. /// The Briggs feature are calculated from the column (freq bin) and row (frame) sums of the extracted spectrogram. /// 1. Gini Index for frame and bin sums. A measure of dispersion. Problem with gini is that its value is dependent on the row or column count. /// We use entropy instead because value not dependent on row or column count because it is normalized. /// For the following meausres of k-central moments, the freq and time values are normalized in 0,1 to width of the event. /// 2. freq-mean /// 3. freq-variance /// 4. freq-skew and kurtosis /// 5. time-mean /// 6. time-variance /// 7. time-skew and kurtosis /// 8. freq-max (normalized) /// 9. time-max (normalized) /// 10. Briggs et al also calculate a 16 value histogram of gradients for each event mask. We do not do that here although we could. /// ... /// NOTE 1: There are differences between our method of noise reduction and Briggs. Briggs does not convert to decibels /// and instead works with power values. He obtains a noise profile from the 20% of frames having the lowest energy sum. /// NOTE 2: To NormaliseMatrixValues for noise, they divide the actual energy by the noise value. This is equivalent to subtraction when working in decibels. /// There are advantages and disadvantages to Briggs method versus ours. In our case, we hve to convert decibel values back to /// energy values when calculating the statistics for the extracted acoustic event. /// NOTE 3: We do not calculate the higher central moments of the time/frequency profiles, i.e. skew and kurtosis. /// Ony mean and standard deviation. /// .. /// NOTE 4: This method assumes that the passed event occurs totally within the passed recording, /// AND that the passed recording is of sufficient duration to obtain reliable BGN noise profile /// BUT not so long as to cause memory constipation. /// </remarks> /// <param name="recording">as type AudioRecording which contains the event</param> /// <param name="temporalTarget">Both start and end bounds - relative to the supplied recording</param> /// <param name="spectralTarget">both bottom and top bounds in Hertz</param> /// <param name="config">parameters that determine the outcome of the analysis</param> /// <param name="segmentStartOffset">How long since the start of the recording this event occurred</param> /// <returns>an instance of EventStatistics</returns> public static EventStatistics AnalyzeAudioEvent( AudioRecording recording, Range <TimeSpan> temporalTarget, Range <double> spectralTarget, EventStatisticsConfiguration config, TimeSpan segmentStartOffset) { var stats = new EventStatistics { EventStartSeconds = temporalTarget.Minimum.TotalSeconds, EventEndSeconds = temporalTarget.Maximum.TotalSeconds, LowFrequencyHertz = spectralTarget.Minimum, HighFrequencyHertz = spectralTarget.Maximum, SegmentDurationSeconds = recording.Duration.TotalSeconds, SegmentStartSeconds = segmentStartOffset.TotalSeconds, }; // temporal target is supplied relative to recording, but not the supplied audio segment // shift coordinates relative to segment var localTemporalTarget = temporalTarget.Shift(-segmentStartOffset); if (!recording .Duration .AsRangeFromZero(Topology.Inclusive) .Contains(localTemporalTarget)) { stats.Error = true; stats.ErrorMessage = $"Audio not long enough ({recording.Duration}) to analyze target ({localTemporalTarget})"; return(stats); } // convert recording to spectrogram int sampleRate = recording.SampleRate; double epsilon = recording.Epsilon; // extract the spectrogram var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recording, config.FrameSize, config.FrameStep); double hertzBinWidth = dspOutput1.FreqBinWidth; var stepDurationInSeconds = config.FrameStep / (double)sampleRate; var startFrame = (int)Math.Ceiling(localTemporalTarget.Minimum.TotalSeconds / stepDurationInSeconds); // subtract 1 frame because want to end before start of end point. var endFrame = (int)Math.Floor(localTemporalTarget.Maximum.TotalSeconds / stepDurationInSeconds) - 1; var bottomBin = (int)Math.Floor(spectralTarget.Minimum / hertzBinWidth); var topBin = (int)Math.Ceiling(spectralTarget.Maximum / hertzBinWidth); // Events can have their high value set to the nyquist. // Since the submatrix call below uses an inclusive upper bound an index out of bounds exception occurs in // these cases. So we just ask for the bin below. if (topBin >= config.FrameSize / 2) { topBin = (config.FrameSize / 2) - 1; } // Convert amplitude spectrogram to deciBels and calculate the dB background noise profile double[,] decibelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput1.AmplitudeSpectrogram, dspOutput1.WindowPower, sampleRate, epsilon); double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram); decibelSpectrogram = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram, spectralDecibelBgn); decibelSpectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram, nhThreshold: 2.0); // extract the required acoustic event var eventMatrix = MatrixTools.Submatrix(decibelSpectrogram, startFrame, bottomBin, endFrame, topBin); // Get the SNR of the event. This is just the max value in the matrix because noise reduced MatrixTools.MinMax(eventMatrix, out _, out double max); stats.SnrDecibels = max; // Now need to convert event matrix back to energy values before calculating other statistics eventMatrix = MatrixTools.Decibels2Power(eventMatrix); var columnAverages = MatrixTools.GetColumnAverages(eventMatrix); var rowAverages = MatrixTools.GetRowAverages(eventMatrix); // calculate the mean and temporal standard deviation in decibels NormalDist.AverageAndSD(rowAverages, out double mean, out double stddev); stats.MeanDecibels = 10 * Math.Log10(mean); stats.TemporalStdDevDecibels = 10 * Math.Log10(stddev); // calculate the frequency standard deviation in decibels NormalDist.AverageAndSD(columnAverages, out mean, out stddev); stats.FreqBinStdDevDecibels = 10 * Math.Log10(stddev); // calculate relative location of the temporal maximum int maxRowId = DataTools.GetMaxIndex(rowAverages); stats.TemporalMaxRelative = maxRowId / (double)rowAverages.Length; // calculate the entropy dispersion/concentration indices stats.TemporalEnergyDistribution = 1 - DataTools.EntropyNormalised(rowAverages); stats.SpectralEnergyDistribution = 1 - DataTools.EntropyNormalised(columnAverages); // calculate the spectral centroid and the dominant frequency double binCentroid = CalculateSpectralCentroid(columnAverages); stats.SpectralCentroid = (int)Math.Round(hertzBinWidth * binCentroid) + (int)spectralTarget.Minimum; int maxColumnId = DataTools.GetMaxIndex(columnAverages); stats.DominantFrequency = (int)Math.Round(hertzBinWidth * maxColumnId) + (int)spectralTarget.Minimum; // remainder of this method is to produce debugging images. Can comment out when not debugging. /* * var normalisedIndex = DataTools.NormaliseMatrixValues(columnAverages); * var image4 = GraphsAndCharts.DrawGraph("columnSums", normalisedIndex, 100); * string path4 = @"C:\SensorNetworks\Output\Sonograms\UnitTestSonograms\columnSums.png"; * image4.Save(path4); * normalisedIndex = DataTools.NormaliseMatrixValues(rowAverages); * image4 = GraphsAndCharts.DrawGraph("rowSums", normalisedIndex, 100); * path4 = @"C:\SensorNetworks\Output\Sonograms\UnitTestSonograms\rowSums.png"; * image4.Save(path4); */ return(stats); }
/// <summary> /// Calculates the following spectrograms as per settings in the Images array in the config file: Towsey.SpectrogramGenerator.yml: /// Waveform. /// DecibelSpectrogram. /// DecibelSpectrogramNoiseReduced. /// CepstralSpectrogram. /// DifferenceSpectrogram. /// AmplitudeSpectrogramLocalContrastNormalization. /// Experimental. /// Comment the config.yml file with a hash, those spectrograms that are not required. /// </summary> /// <param name="sourceRecording">The name of the original recording.</param> /// <param name="config">Contains parameter info to make spectrograms.</param> /// <param name="sourceRecordingName">.Name of source recording. Required only spectrogram labels.</param> public static AudioToSonogramResult GenerateSpectrogramImages( FileInfo sourceRecording, SpectrogramGeneratorConfig config, string sourceRecordingName) { //int signalLength = recordingSegment.WavReader.GetChannel(0).Length; var recordingSegment = new AudioRecording(sourceRecording.FullName); int sampleRate = recordingSegment.WavReader.SampleRate; var result = new AudioToSonogramResult(); var requestedImageTypes = config.Images ?? new[] { SpectrogramImageType.DecibelSpectrogram }; var @do = requestedImageTypes.ToHashSet(); int frameSize = config.GetIntOrNull("FrameLength") ?? 512; int frameStep = config.GetIntOrNull("FrameStep") ?? 441; // must calculate this because used later on. double frameOverlap = (frameSize - frameStep) / (double)frameSize; // Default noiseReductionType = Standard var bgNoiseThreshold = config.BgNoiseThreshold; // threshold for drawing the difference spectrogram var differenceThreshold = config.DifferenceThreshold; // EXTRACT ENVELOPE and SPECTROGRAM FROM RECORDING SEGMENT var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recordingSegment, frameSize, frameStep); var sonoConfig = new SonogramConfig() { epsilon = recordingSegment.Epsilon, SampleRate = sampleRate, WindowSize = frameSize, WindowStep = frameStep, WindowOverlap = frameOverlap, WindowPower = dspOutput1.WindowPower, Duration = recordingSegment.Duration, NoiseReductionType = NoiseReductionType.Standard, NoiseReductionParameter = bgNoiseThreshold, }; var images = new Dictionary <SpectrogramImageType, Image <Rgb24> >(requestedImageTypes.Length); // IMAGE 1) draw the WAVEFORM if (@do.Contains(Waveform)) { var minValues = dspOutput1.MinFrameValues; var maxValues = dspOutput1.MaxFrameValues; int height = config.WaveformHeight; var waveformImage = GetWaveformImage(minValues, maxValues, height); // add in the title bar and time scales. string title = $"WAVEFORM - {sourceRecordingName} (min value={dspOutput1.MinSignalValue:f3}, max value={dspOutput1.MaxSignalValue:f3})"; var titleBar = BaseSonogram.DrawTitleBarOfGrayScaleSpectrogram( title, waveformImage.Width, ImageTags[Waveform]); var startTime = TimeSpan.Zero; var xAxisTicInterval = TimeSpan.FromSeconds(1); TimeSpan xAxisPixelDuration = TimeSpan.FromSeconds(frameStep / (double)sampleRate); var labelInterval = TimeSpan.FromSeconds(5); waveformImage = BaseSonogram.FrameSonogram( waveformImage, titleBar, startTime, xAxisTicInterval, xAxisPixelDuration, labelInterval); images.Add(Waveform, waveformImage); } // Draw various decibel spectrograms var decibelTypes = new[] { SpectrogramImageType.DecibelSpectrogram, DecibelSpectrogramNoiseReduced, DifferenceSpectrogram, Experimental }; if (@do.Overlaps(decibelTypes)) { // disable noise removal for first two spectrograms var disabledNoiseReductionType = sonoConfig.NoiseReductionType; sonoConfig.NoiseReductionType = NoiseReductionType.None; //Get the decibel spectrogram var decibelSpectrogram = new SpectrogramStandard(sonoConfig, dspOutput1.AmplitudeSpectrogram); result.DecibelSpectrogram = decibelSpectrogram; double[,] dbSpectrogramData = (double[, ])decibelSpectrogram.Data.Clone(); // IMAGE 2) Display the DecibelSpectrogram if (@do.Contains(SpectrogramImageType.DecibelSpectrogram)) { images.Add( SpectrogramImageType.DecibelSpectrogram, decibelSpectrogram.GetImageFullyAnnotated( $"DECIBEL SPECTROGRAM ({sourceRecordingName})", ImageTags[SpectrogramImageType.DecibelSpectrogram])); } if (@do.Overlaps(new[] { DecibelSpectrogramNoiseReduced, Experimental, CepstralSpectrogram })) { sonoConfig.NoiseReductionType = disabledNoiseReductionType; sonoConfig.NoiseReductionParameter = bgNoiseThreshold; double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram.Data); decibelSpectrogram.Data = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram.Data, spectralDecibelBgn); decibelSpectrogram.Data = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram.Data, nhThreshold: bgNoiseThreshold); // IMAGE 3) DecibelSpectrogram - noise reduced if (@do.Contains(DecibelSpectrogramNoiseReduced)) { images.Add( DecibelSpectrogramNoiseReduced, decibelSpectrogram.GetImageFullyAnnotated( $"DECIBEL SPECTROGRAM + Lamel noise subtraction. ({sourceRecordingName})", ImageTags[DecibelSpectrogramNoiseReduced])); } // IMAGE 4) EXPERIMENTAL Spectrogram if (@do.Contains(Experimental)) { sonoConfig.NoiseReductionType = disabledNoiseReductionType; images.Add( Experimental, GetDecibelSpectrogram_Ridges( dbSpectrogramData, decibelSpectrogram, sourceRecordingName)); } } // IMAGE 5) draw difference spectrogram. This is derived from the original decibel spectrogram if (@do.Contains(DifferenceSpectrogram)) { //var differenceThreshold = configInfo.GetDoubleOrNull("DifferenceThreshold") ?? 3.0; var differenceImage = GetDifferenceSpectrogram(dbSpectrogramData, differenceThreshold); differenceImage = BaseSonogram.GetImageAnnotatedWithLinearHertzScale( differenceImage, sampleRate, frameStep, $"DECIBEL DIFFERENCE SPECTROGRAM ({sourceRecordingName})", ImageTags[DifferenceSpectrogram]); images.Add(DifferenceSpectrogram, differenceImage); } } // IMAGE 6) Cepstral Spectrogram if (@do.Contains(CepstralSpectrogram)) { images.Add( CepstralSpectrogram, GetCepstralSpectrogram(sonoConfig, recordingSegment, sourceRecordingName)); } // IMAGE 7) AmplitudeSpectrogram_LocalContrastNormalization if (@do.Contains(AmplitudeSpectrogramLocalContrastNormalization)) { var neighborhoodSeconds = config.NeighborhoodSeconds; var lcnContrastParameter = config.LcnContrastLevel; images.Add( AmplitudeSpectrogramLocalContrastNormalization, GetLcnSpectrogram( sonoConfig, recordingSegment, sourceRecordingName, neighborhoodSeconds, lcnContrastParameter)); } // now pick and combine images in order user specified var sortedImages = requestedImageTypes.Select(x => images[x]); // COMBINE THE SPECTROGRAM IMAGES result.CompositeImage = ImageTools.CombineImagesVertically(sortedImages.ToArray()); return(result); }
/// <summary> /// Calculates the following spectrograms as per content of config.yml file: /// Waveform: true. /// DifferenceSpectrogram: true. /// DecibelSpectrogram: true. /// DecibelSpectrogram_NoiseReduced: true. /// DecibelSpectrogram_Ridges: true. /// AmplitudeSpectrogram_LocalContrastNormalization: true. /// SoxSpectrogram: false. /// Experimental: true. /// </summary> /// <param name="sourceRecording">The name of the original recording.</param> /// <param name="configInfo">Contains parameter info to make spectrograms.</param> /// <param name="sourceRecordingName">.Name of source recording. Required only spectrogram labels.</param> public static AudioToSonogramResult GenerateSpectrogramImages( FileInfo sourceRecording, AnalyzerConfig configInfo, string sourceRecordingName) { //int signalLength = recordingSegment.WavReader.GetChannel(0).Length; var recordingSegment = new AudioRecording(sourceRecording.FullName); int sampleRate = recordingSegment.WavReader.SampleRate; var result = new AudioToSonogramResult(); // init the image stack var list = new List <Image>(); bool doWaveForm = configInfo.GetBoolOrNull("Waveform") ?? false; bool doDecibelSpectrogram = configInfo.GetBoolOrNull("DecibelSpectrogram") ?? false; bool doNoiseReducedSpectrogram = configInfo.GetBoolOrNull("DecibelSpectrogram_NoiseReduced") ?? true; bool doDifferenceSpectrogram = configInfo.GetBoolOrNull("DifferenceSpectrogram") ?? false; bool doLcnSpectrogram = configInfo.GetBoolOrNull("AmplitudeSpectrogram_LocalContrastNormalization") ?? false; bool doCepstralSpectrogram = configInfo.GetBoolOrNull("CepstralSpectrogram") ?? false; bool doExperimentalSpectrogram = configInfo.GetBoolOrNull("Experimental") ?? false; //Don't do SOX spectrogram. //bool doSoxSpectrogram = configInfo.GetBool("SoxSpectrogram"); int frameSize = configInfo.GetIntOrNull("FrameLength") ?? 512; int frameStep = configInfo.GetIntOrNull("FrameStep") ?? 0; // must calculate this because used later on. double frameOverlap = (frameSize - frameStep) / (double)frameSize; // Default noiseReductionType = Standard var bgNoiseThreshold = configInfo.GetDoubleOrNull("BgNoiseThreshold") ?? 3.0; // EXTRACT ENVELOPE and SPECTROGRAM FROM RECORDING SEGMENT var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recordingSegment, frameSize, frameStep); var sonoConfig = new SonogramConfig() { epsilon = recordingSegment.Epsilon, SampleRate = sampleRate, WindowSize = frameSize, WindowStep = frameStep, WindowOverlap = frameOverlap, WindowPower = dspOutput1.WindowPower, Duration = recordingSegment.Duration, NoiseReductionType = NoiseReductionType.Standard, NoiseReductionParameter = bgNoiseThreshold, }; // IMAGE 1) draw the WAVEFORM if (doWaveForm) { var minValues = dspOutput1.MinFrameValues; var maxValues = dspOutput1.MaxFrameValues; int height = configInfo.GetIntOrNull("WaveformHeight") ?? 180; var waveformImage = GetWaveformImage(minValues, maxValues, height); // add in the title bar and time scales. string title = $"WAVEFORM - {sourceRecordingName} (min value={dspOutput1.MinSignalValue:f3}, max value={dspOutput1.MaxSignalValue:f3})"; var titleBar = BaseSonogram.DrawTitleBarOfGrayScaleSpectrogram(title, waveformImage.Width); var startTime = TimeSpan.Zero; var xAxisTicInterval = TimeSpan.FromSeconds(1); TimeSpan xAxisPixelDuration = TimeSpan.FromSeconds(frameStep / (double)sampleRate); var labelInterval = TimeSpan.FromSeconds(5); waveformImage = BaseSonogram.FrameSonogram(waveformImage, titleBar, startTime, xAxisTicInterval, xAxisPixelDuration, labelInterval); list.Add(waveformImage); } // Draw various decibel spectrograms if (doDecibelSpectrogram || doNoiseReducedSpectrogram || doDifferenceSpectrogram || doExperimentalSpectrogram) { // disable noise removal for first spectrogram var disabledNoiseReductionType = sonoConfig.NoiseReductionType; sonoConfig.NoiseReductionType = NoiseReductionType.None; //Get the decibel spectrogram var decibelSpectrogram = new SpectrogramStandard(sonoConfig, dspOutput1.AmplitudeSpectrogram); result.DecibelSpectrogram = decibelSpectrogram; double[,] dbSpectrogramData = (double[, ])decibelSpectrogram.Data.Clone(); // IMAGE 2) DecibelSpectrogram if (doDecibelSpectrogram) { var image3 = decibelSpectrogram.GetImageFullyAnnotated($"DECIBEL SPECTROGRAM ({sourceRecordingName})"); list.Add(image3); } if (doNoiseReducedSpectrogram || doExperimentalSpectrogram || doDifferenceSpectrogram) { sonoConfig.NoiseReductionType = disabledNoiseReductionType; sonoConfig.NoiseReductionParameter = bgNoiseThreshold; double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram.Data); decibelSpectrogram.Data = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram.Data, spectralDecibelBgn); decibelSpectrogram.Data = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram.Data, nhThreshold: bgNoiseThreshold); // IMAGE 3) DecibelSpectrogram - noise reduced if (doNoiseReducedSpectrogram) { var image4 = decibelSpectrogram.GetImageFullyAnnotated($"DECIBEL SPECTROGRAM + Lamel noise subtraction. ({sourceRecordingName})"); list.Add(image4); } // IMAGE 4) EXPERIMENTAL Spectrogram if (doExperimentalSpectrogram) { sonoConfig.NoiseReductionType = disabledNoiseReductionType; var image5 = GetDecibelSpectrogram_Ridges(dbSpectrogramData, decibelSpectrogram, sourceRecordingName); list.Add(image5); } // IMAGE 5) draw difference spectrogram if (doDifferenceSpectrogram) { var differenceThreshold = configInfo.GetDoubleOrNull("DifferenceThreshold") ?? 3.0; var image6 = GetDifferenceSpectrogram(dbSpectrogramData, differenceThreshold); image6 = BaseSonogram.GetImageAnnotatedWithLinearHertzScale(image6, sampleRate, frameStep, $"DECIBEL DIFFERENCE SPECTROGRAM ({sourceRecordingName})"); list.Add(image6); } } } // IMAGE 6) Cepstral Spectrogram if (doCepstralSpectrogram) { var image6 = GetCepstralSpectrogram(sonoConfig, recordingSegment, sourceRecordingName); list.Add(image6); } // 7) AmplitudeSpectrogram_LocalContrastNormalization if (doLcnSpectrogram) { var neighbourhoodSeconds = configInfo.GetDoubleOrNull("NeighbourhoodSeconds") ?? 0.5; var lcnContrastParameter = configInfo.GetDoubleOrNull("LcnContrastLevel") ?? 0.4; var image8 = GetLcnSpectrogram(sonoConfig, recordingSegment, sourceRecordingName, neighbourhoodSeconds, lcnContrastParameter); list.Add(image8); } // 8) SOX SPECTROGRAM //if (doSoxSpectrogram) //{ //Log.Warn("SoX spectrogram set to true but is ignored when running as an IAnalyzer"); // The following parameters were once used to implement a sox spectrogram. //bool makeSoxSonogram = configuration.GetBoolOrNull(AnalysisKeys.MakeSoxSonogram) ?? false; //configDict[AnalysisKeys.SonogramTitle] = configuration[AnalysisKeys.SonogramTitle] ?? "Sonogram"; //configDict[AnalysisKeys.SonogramComment] = configuration[AnalysisKeys.SonogramComment] ?? "Sonogram produced using SOX"; //configDict[AnalysisKeys.SonogramColored] = configuration[AnalysisKeys.SonogramColored] ?? "false"; //configDict[AnalysisKeys.SonogramQuantisation] = configuration[AnalysisKeys.SonogramQuantisation] ?? "128"; //configDict[AnalysisKeys.AddTimeScale] = configuration[AnalysisKeys.AddTimeScale] ?? "true"; //configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true"; //configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true"; // var soxFile = new FileInfo(Path.Combine(output.FullName, sourceName + "SOX.png")); // SpectrogramTools.MakeSonogramWithSox(sourceRecording, configDict, path2SoxSpectrogram); // list.Add(image7); //} // COMBINE THE SPECTROGRAM IMAGES result.CompositeImage = ImageTools.CombineImagesVertically(list); return(result); }
public static IndexCalculateResult Analysis( AudioRecording recording, TimeSpan subsegmentOffsetTimeSpan, Dictionary <string, IndexProperties> indexProperties, int sampleRateOfOriginalAudioFile, TimeSpan segmentStartOffset, IndexCalculateConfig config, bool returnSonogramInfo = false) { // returnSonogramInfo = true; // if debugging double epsilon = recording.Epsilon; int signalLength = recording.WavReader.GetChannel(0).Length; int sampleRate = recording.WavReader.SampleRate; var segmentDuration = TimeSpan.FromSeconds(recording.WavReader.Time.TotalSeconds); var indexCalculationDuration = config.IndexCalculationDurationTimeSpan; int nyquist = sampleRate / 2; // Get FRAME parameters for the calculation of Acoustic Indices //WARNING: DO NOT USE Frame Overlap when calculating acoustic indices. // It yields ACI, BGN, POW and EVN results that are significantly different from the default. // I have not had time to check if the difference is meaningful. Best to avoid. //int frameSize = (int?)config[AnalysisKeys.FrameLength] ?? IndexCalculateConfig.DefaultWindowSize; int frameSize = config.FrameLength; int frameStep = frameSize; // that is, windowOverlap = zero double frameStepDuration = frameStep / (double)sampleRate; // fraction of a second var frameStepTimeSpan = TimeSpan.FromTicks((long)(frameStepDuration * TimeSpan.TicksPerSecond)); int midFreqBound = config.MidFreqBound; int lowFreqBound = config.LowFreqBound; int freqBinCount = frameSize / 2; // double freqBinWidth = recording.Nyquist / (double)freqBinCount; // get duration in seconds and sample count and frame count double subsegmentDurationInSeconds = indexCalculationDuration.TotalSeconds; int subsegmentSampleCount = (int)(subsegmentDurationInSeconds * sampleRate); double subsegmentFrameCount = subsegmentSampleCount / (double)frameStep; subsegmentFrameCount = (int)Math.Ceiling(subsegmentFrameCount); // In order not to lose the last fractional frame, round up the frame number // and get the exact number of samples in the integer number of frames. // Do this because when IndexCalculationDuration = 100ms, the number of frames is only 8. subsegmentSampleCount = (int)(subsegmentFrameCount * frameStep); // get start and end samples of the subsegment and noise segment double localOffsetInSeconds = subsegmentOffsetTimeSpan.TotalSeconds - segmentStartOffset.TotalSeconds; int startSample = (int)(localOffsetInSeconds * sampleRate); int endSample = startSample + subsegmentSampleCount - 1; // Default behaviour: set SUBSEGMENT = total recording var subsegmentRecording = recording; // But if the indexCalculationDuration < segmentDuration if (indexCalculationDuration < segmentDuration) { // minimum samples needed to calculate acoustic indices. This value was chosen somewhat arbitrarily. // It allowes for case where IndexCalculationDuration = 100ms which is approx 8 frames int minimumViableSampleCount = frameSize * 8; int availableSignal = signalLength - startSample; // if (the required audio is beyond recording OR insufficient for analysis) then backtrack. if (availableSignal < minimumViableSampleCount) { // Back-track so we can fill a whole result. // This is a silent correction, equivalent to having a segment overlap for the last segment. var oldStart = startSample; startSample = signalLength - subsegmentSampleCount; endSample = signalLength; Logger.Trace(" Backtrack subsegment to fill missing data from imperfect audio cuts because not enough samples available. " + (oldStart - startSample) + " samples overlap."); } var subsamples = DataTools.Subarray(recording.WavReader.Samples, startSample, subsegmentSampleCount); var wr = new Acoustics.Tools.Wav.WavReader(subsamples, 1, 16, sampleRate); subsegmentRecording = new AudioRecording(wr); } // INITIALISE a RESULTS STRUCTURE TO return // initialize a result object in which to store SummaryIndexValues and SpectralIndexValues etc. var result = new IndexCalculateResult(freqBinCount, indexProperties, indexCalculationDuration, subsegmentOffsetTimeSpan, config); SummaryIndexValues summaryIndices = result.SummaryIndexValues; SpectralIndexValues spectralIndices = result.SpectralIndexValues; // set up default spectrogram to return result.Sg = returnSonogramInfo ? GetSonogram(recording, windowSize: 1024) : null; result.Hits = null; result.TrackScores = new List <Plot>(); // ################################## FINSIHED SET-UP // ################################## NOW GET THE AMPLITUDE SPECTORGRAMS // EXTRACT ENVELOPE and SPECTROGRAM FROM SUBSEGMENT var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(subsegmentRecording, frameSize, frameStep); // Select band according to min and max bandwidth int minBand = (int)(dspOutput1.AmplitudeSpectrogram.GetLength(1) * config.MinBandWidth); int maxBand = (int)(dspOutput1.AmplitudeSpectrogram.GetLength(1) * config.MaxBandWidth) - 1; dspOutput1.AmplitudeSpectrogram = MatrixTools.Submatrix( dspOutput1.AmplitudeSpectrogram, 0, minBand, dspOutput1.AmplitudeSpectrogram.GetLength(0) - 1, maxBand); // TODO: Michael to review whether bandwidth filter should be moved to DSP_Frames?? // Recalculate NyquistBin and FreqBinWidth, because they change with band selection //dspOutput1.NyquistBin = dspOutput1.AmplitudeSpectrogram.GetLength(1) - 1; //dspOutput1.FreqBinWidth = sampleRate / (double)dspOutput1.AmplitudeSpectrogram.GetLength(1) / 2; // Linear or Octave or Mel frequency scale? Set Linear as default. var freqScale = new FrequencyScale(nyquist: nyquist, frameSize: frameSize, hertzGridInterval: 1000); var freqScaleType = config.FrequencyScale; bool octaveScale = freqScaleType == FreqScaleType.Linear125Octaves7Tones28Nyquist32000; bool melScale = freqScaleType == FreqScaleType.Mel; if (octaveScale) { // only allow one octave scale at the moment - for Jasco marine recordings. // ASSUME fixed Occtave scale - USEFUL ONLY FOR JASCO 64000sr MARINE RECORDINGS // If you wish to use other octave scale types then need to put in the config file and and set up recovery here. freqScale = new FrequencyScale(FreqScaleType.Linear125Octaves7Tones28Nyquist32000); // Recalculate the spectrogram according to octave scale. This option works only when have high SR recordings. dspOutput1.AmplitudeSpectrogram = OctaveFreqScale.AmplitudeSpectra( dspOutput1.AmplitudeSpectrogram, dspOutput1.WindowPower, sampleRate, epsilon, freqScale); dspOutput1.NyquistBin = dspOutput1.AmplitudeSpectrogram.GetLength(1) - 1; // ASSUMPTION!!! Nyquist is in top Octave bin - not necessarily true!! } else if (melScale) { int minFreq = 0; int maxFreq = recording.Nyquist; dspOutput1.AmplitudeSpectrogram = MFCCStuff.MelFilterBank( dspOutput1.AmplitudeSpectrogram, config.MelScale, recording.Nyquist, minFreq, maxFreq); dspOutput1.NyquistBin = dspOutput1.AmplitudeSpectrogram.GetLength(1) - 1; // TODO: This doesn't make any sense, since the frequency width changes for each bin. Probably need to set this to NaN. // TODO: Whatever uses this value below, should probably be changed to not be depending on it. dspOutput1.FreqBinWidth = sampleRate / (double)dspOutput1.AmplitudeSpectrogram.GetLength(1) / 2; } // NOW EXTRACT SIGNAL FOR BACKGROUND NOISE CALCULATION // If the index calculation duration >= 30 seconds, then calculate BGN from the existing segment of recording. bool doSeparateBgnNoiseCalculation = indexCalculationDuration.TotalSeconds + (2 * config.BgNoiseBuffer.TotalSeconds) < segmentDuration.TotalSeconds / 2; var dspOutput2 = dspOutput1; if (doSeparateBgnNoiseCalculation) { // GET a longer SUBSEGMENT FOR NOISE calculation with 5 sec buffer on either side. // If the index calculation duration is shorter than 30 seconds, then need to calculate BGN noise from a longer length of recording // i.e. need to add noiseBuffer either side. Typical noiseBuffer value = 5 seconds int sampleBuffer = (int)(config.BgNoiseBuffer.TotalSeconds * sampleRate); var bgnRecording = AudioRecording.GetRecordingSubsegment(recording, startSample, endSample, sampleBuffer); // EXTRACT ENVELOPE and SPECTROGRAM FROM BACKGROUND NOISE SUBSEGMENT dspOutput2 = DSP_Frames.ExtractEnvelopeAndFfts(bgnRecording, frameSize, frameStep); // If necessary, recalculate the spectrogram according to octave scale. This option works only when have high SR recordings. if (octaveScale) { // ASSUME fixed Occtave scale - USEFUL ONLY FOR JASCO 64000sr MARINE RECORDINGS // If you wish to use other octave scale types then need to put in the config file and and set up recovery here. dspOutput2.AmplitudeSpectrogram = OctaveFreqScale.AmplitudeSpectra( dspOutput2.AmplitudeSpectrogram, dspOutput2.WindowPower, sampleRate, epsilon, freqScale); dspOutput2.NyquistBin = dspOutput2.AmplitudeSpectrogram.GetLength(1) - 1; // ASSUMPTION!!! Nyquist is in top Octave bin - not necessarily true!! } } // ###################################### BEGIN CALCULATION OF INDICES ################################## // (A) ################################## EXTRACT SUMMARY INDICES FROM THE SIGNAL WAVEFORM ################################## // average absolute value over the minute recording - not useful // double[] avAbsolute = dspOutput1.Average; double[] signalEnvelope = dspOutput1.Envelope; double avgSignalEnvelope = signalEnvelope.Average(); // 10 times log of amplitude squared summaryIndices.AvgSignalAmplitude = 20 * Math.Log10(avgSignalEnvelope); // Deal with case where the signal waveform is continuous flat with values < 0.001. Has happened!! // Although signal appears zero, this condition is required. if (avgSignalEnvelope < 0.0001) { Logger.Debug("Segment skipped because avSignalEnvelope is < 0.001!"); summaryIndices.ZeroSignal = 1.0; return(result); } // i. Check for clipping and high amplitude rates per second summaryIndices.HighAmplitudeIndex = dspOutput1.HighAmplitudeCount / subsegmentDurationInSeconds; summaryIndices.ClippingIndex = dspOutput1.ClipCount / subsegmentDurationInSeconds; // ii. Calculate bg noise in dB // Convert signal envelope to dB and subtract background noise. Default noise SD to calculate threshold = ZERO double signalBgn = NoiseRemovalModal.CalculateBackgroundNoise(dspOutput2.Envelope); summaryIndices.BackgroundNoise = signalBgn; // iii: FRAME ENERGIES - convert signal to decibels and subtract background noise. double[] dBEnvelope = SNR.Signal2Decibels(dspOutput1.Envelope); double[] dBEnvelopeSansNoise = SNR.SubtractAndTruncate2Zero(dBEnvelope, signalBgn); // iv: ACTIVITY for NOISE REDUCED SIGNAL ENVELOPE // Calculate fraction of frames having acoustic activity var activity = ActivityAndCover.CalculateActivity(dBEnvelopeSansNoise, frameStepTimeSpan); summaryIndices.Activity = activity.FractionOfActiveFrames; // v. average number of events per second whose duration > one frame // average event duration in milliseconds - no longer calculated //summaryIndices.AvgEventDuration = activity.avEventDuration; summaryIndices.EventsPerSecond = activity.EventCount / subsegmentDurationInSeconds; // vi. Calculate SNR and active frames SNR summaryIndices.Snr = dBEnvelopeSansNoise.Max(); summaryIndices.AvgSnrOfActiveFrames = activity.ActiveAvDb; // vii. ENTROPY of ENERGY ENVELOPE -- 1-Ht because want measure of concentration of acoustic energy. double entropy = DataTools.EntropyNormalised(DataTools.SquareValues(signalEnvelope)); summaryIndices.TemporalEntropy = 1 - entropy; // Note that the spectrogram has had the DC bin removed. i.e. has only 256 columns. double[,] amplitudeSpectrogram = dspOutput1.AmplitudeSpectrogram; // get amplitude spectrogram. // CALCULATE various NDSI (Normalised difference soundscape Index) FROM THE AMPLITUDE SPECTROGRAM // These options proved to be highly correlated. Therefore only use tuple.Item 1 which derived from Power Spectral Density. var tuple3 = SpectrogramTools.CalculateAvgSpectrumAndVarianceSpectrumFromAmplitudeSpectrogram(amplitudeSpectrogram); summaryIndices.Ndsi = SpectrogramTools.CalculateNdsi(tuple3.Item1, sampleRate, 1000, 2000, 8000); // (B) ################################## EXTRACT OSC SPECTRAL INDEX DIRECTLY FROM THE RECORDING ################################## // Get the oscillation spectral index OSC separately from signal because need a different frame size etc. var sampleLength = Oscillations2014.DefaultSampleLength; var frameLength = Oscillations2014.DefaultFrameLength; var sensitivity = Oscillations2014.DefaultSensitivityThreshold; var spectralIndexShort = Oscillations2014.GetSpectralIndex_Osc(subsegmentRecording, frameLength, sampleLength, sensitivity); // double length of the vector because want to work with 256 element vector for LDFC purposes spectralIndices.OSC = DataTools.VectorDoubleLengthByAverageInterpolation(spectralIndexShort); // (C) ################################## EXTRACT SPECTRAL INDICES FROM THE AMPLITUDE SPECTROGRAM ################################## // i: CALCULATE SPECTRUM OF THE SUM OF FREQ BIN AMPLITUDES - used for later calculation of ACI spectralIndices.SUM = MatrixTools.SumColumns(amplitudeSpectrogram); // Calculate lower and upper boundary bin ids. // Boundary between low & mid frequency bands is to avoid low freq bins containing anthropogenic noise. These biased index values away from biophony. // Boundary of upper bird-band is to avoid high freq artefacts due to mp3. int lowerBinBound = (int)Math.Ceiling(lowFreqBound / dspOutput1.FreqBinWidth); int middleBinBound = (int)Math.Ceiling(midFreqBound / dspOutput1.FreqBinWidth); // calculate number of freq bins in the bird-band. int midBandBinCount = middleBinBound - lowerBinBound + 1; if (octaveScale) { // the above frequency bin bounds do not apply with octave scale. Need to recalculate them suitable for Octave scale recording. lowFreqBound = freqScale.LinearBound; lowerBinBound = freqScale.GetBinIdForHerzValue(lowFreqBound); midFreqBound = 8000; // This value appears suitable for Jasco Marine recordings. Not much happens above 8kHz. //middleBinBound = freqScale.GetBinIdForHerzValue(midFreqBound); middleBinBound = freqScale.GetBinIdInReducedSpectrogramForHerzValue(midFreqBound); midBandBinCount = middleBinBound - lowerBinBound + 1; } // IFF there has been UP-SAMPLING, calculate bin of the original audio nyquist. this will be less than SR/2. // original sample rate can be anything 11.0-44.1 kHz. int originalNyquist = sampleRateOfOriginalAudioFile / 2; // if upsampling has been done if (dspOutput1.NyquistFreq > originalNyquist) { dspOutput1.NyquistFreq = originalNyquist; dspOutput1.NyquistBin = (int)Math.Floor(originalNyquist / dspOutput1.FreqBinWidth); // note that binwidth does not change } // ii: CALCULATE THE ACOUSTIC COMPLEXITY INDEX spectralIndices.DIF = AcousticComplexityIndex.SumOfAmplitudeDifferences(amplitudeSpectrogram); double[] aciSpectrum = AcousticComplexityIndex.CalculateAci(amplitudeSpectrogram); spectralIndices.ACI = aciSpectrum; // remove low freq band of ACI spectrum and store average ACI value double[] reducedAciSpectrum = DataTools.Subarray(aciSpectrum, lowerBinBound, midBandBinCount); summaryIndices.AcousticComplexity = reducedAciSpectrum.Average(); // iii: CALCULATE the H(t) or Temporal ENTROPY Spectrum and then reverse the values i.e. calculate 1-Ht for energy concentration double[] temporalEntropySpectrum = AcousticEntropy.CalculateTemporalEntropySpectrum(amplitudeSpectrogram); for (int i = 0; i < temporalEntropySpectrum.Length; i++) { temporalEntropySpectrum[i] = 1 - temporalEntropySpectrum[i]; } spectralIndices.ENT = temporalEntropySpectrum; // iv: remove background noise from the amplitude spectrogram // First calculate the noise profile from the amplitude sepctrogram double[] spectralAmplitudeBgn = NoiseProfile.CalculateBackgroundNoise(dspOutput2.AmplitudeSpectrogram); amplitudeSpectrogram = SNR.TruncateBgNoiseFromSpectrogram(amplitudeSpectrogram, spectralAmplitudeBgn); // AMPLITUDE THRESHOLD for smoothing background, nhThreshold, assumes background noise ranges around -40dB. // This value corresponds to approximately 6dB above backgorund. amplitudeSpectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(amplitudeSpectrogram, nhThreshold: 0.015); ////ImageTools.DrawMatrix(spectrogramData, @"C:\SensorNetworks\WavFiles\Crows\image.png", false); ////DataTools.writeBarGraph(modalValues); result.AmplitudeSpectrogram = amplitudeSpectrogram; // v: ENTROPY OF AVERAGE SPECTRUM & VARIANCE SPECTRUM - at this point the spectrogram is a noise reduced amplitude spectrogram var tuple = AcousticEntropy.CalculateSpectralEntropies(amplitudeSpectrogram, lowerBinBound, midBandBinCount); // ENTROPY of spectral averages - Reverse the values i.e. calculate 1-Hs and 1-Hv, and 1-Hcov for energy concentration summaryIndices.EntropyOfAverageSpectrum = 1 - tuple.Item1; // ENTROPY of spectrum of Variance values summaryIndices.EntropyOfVarianceSpectrum = 1 - tuple.Item2; // ENTROPY of spectrum of Coefficient of Variation values summaryIndices.EntropyOfCoVSpectrum = 1 - tuple.Item3; // vi: ENTROPY OF DISTRIBUTION of maximum SPECTRAL PEAKS. // First extract High band SPECTROGRAM which is now noise reduced double entropyOfPeaksSpectrum = AcousticEntropy.CalculateEntropyOfSpectralPeaks(amplitudeSpectrogram, lowerBinBound, middleBinBound); summaryIndices.EntropyOfPeaksSpectrum = 1 - entropyOfPeaksSpectrum; // ###################################################################################################################################################### // (C) ################################## EXTRACT SPECTRAL INDICES FROM THE DECIBEL SPECTROGRAM ################################## // i: Convert amplitude spectrogram to deciBels and calculate the dB background noise profile double[,] deciBelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput2.AmplitudeSpectrogram, dspOutput2.WindowPower, sampleRate, epsilon); double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(deciBelSpectrogram); spectralIndices.BGN = spectralDecibelBgn; // ii: Calculate the noise reduced decibel spectrogram derived from segment recording. // REUSE the var decibelSpectrogram but this time using dspOutput1. deciBelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput1.AmplitudeSpectrogram, dspOutput1.WindowPower, sampleRate, epsilon); deciBelSpectrogram = SNR.TruncateBgNoiseFromSpectrogram(deciBelSpectrogram, spectralDecibelBgn); deciBelSpectrogram = SNR.RemoveNeighbourhoodBackgroundNoise(deciBelSpectrogram, nhThreshold: 2.0); // iii: CALCULATE noise reduced AVERAGE DECIBEL SPECTRUM spectralIndices.PMN = SpectrogramTools.CalculateAvgDecibelSpectrumFromDecibelSpectrogram(deciBelSpectrogram); // iv: CALCULATE SPECTRAL COVER. // NOTE: at this point, decibelSpectrogram is noise reduced. All values >= 0.0 // FreqBinWidth can be accessed, if required, through dspOutput1.FreqBinWidth double dBThreshold = ActivityAndCover.DefaultActivityThresholdDb; // dB THRESHOLD for calculating spectral coverage var spActivity = ActivityAndCover.CalculateSpectralEvents(deciBelSpectrogram, dBThreshold, frameStepTimeSpan, lowerBinBound, middleBinBound); spectralIndices.CVR = spActivity.CoverSpectrum; spectralIndices.EVN = spActivity.EventSpectrum; summaryIndices.HighFreqCover = spActivity.HighFreqBandCover; summaryIndices.MidFreqCover = spActivity.MidFreqBandCover; summaryIndices.LowFreqCover = spActivity.LowFreqBandCover; // ###################################################################################################################################################### // v: CALCULATE SPECTRAL PEAK TRACKS and RIDGE indices. // NOTE: at this point, the var decibelSpectrogram is noise reduced. i.e. all its values >= 0.0 // Detecting ridges or spectral peak tracks requires using a 5x5 mask which has edge effects. // This becomes significant if we have a short indexCalculationDuration. // Consequently if the indexCalculationDuration < 10 seconds then we revert back to the recording and cut out a recording segment that includes // a buffer for edge effects. In most cases however, we can just use the decibel spectrogram already calculated and ignore the edge effects. double peakThreshold = 6.0; //dB SpectralPeakTracks sptInfo; if (indexCalculationDuration.TotalSeconds < 10.0) { // calculate a new decibel spectrogram sptInfo = SpectralPeakTracks.CalculateSpectralPeakTracks(recording, startSample, endSample, frameSize, octaveScale, peakThreshold); } else { // use existing decibel spectrogram sptInfo = new SpectralPeakTracks(deciBelSpectrogram, peakThreshold); } spectralIndices.SPT = sptInfo.SptSpectrum; spectralIndices.RHZ = sptInfo.RhzSpectrum; spectralIndices.RVT = sptInfo.RvtSpectrum; spectralIndices.RPS = sptInfo.RpsSpectrum; spectralIndices.RNG = sptInfo.RngSpectrum; summaryIndices.SptDensity = sptInfo.TrackDensity; // these are two other indices that I tried but they do not seem to add anything of interest. //summaryIndices.AvgSptDuration = sptInfo.AvTrackDuration; //summaryIndices.SptPerSecond = sptInfo.TotalTrackCount / subsegmentSecondsDuration; // ###################################################################################################################################################### // vi: CLUSTERING - FIRST DETERMINE IF IT IS WORTH DOING // return if (activeFrameCount too small || eventCount == 0 || short index calc duration) because no point doing clustering if (activity.ActiveFrameCount <= 2 || Math.Abs(activity.EventCount) < 0.01 || indexCalculationDuration.TotalSeconds < 15) { // IN ADDITION return if indexCalculationDuration < 15 seconds because no point doing clustering on short time segment // NOTE: Activity was calculated with 3dB threshold AFTER backgroundnoise removal. //summaryIndices.AvgClusterDuration = TimeSpan.Zero; summaryIndices.ClusterCount = 0; summaryIndices.ThreeGramCount = 0; return(result); } // YES WE WILL DO CLUSTERING! to determine cluster count (spectral diversity) and spectral persistence. // Only use midband decibel SPECTRUM. In June 2016, the mid-band (i.e. the bird-band) was set to lowerBound=1000Hz, upperBound=8000hz. // Actually do clustering of binary spectra. Must first threshold double binaryThreshold = SpectralClustering.DefaultBinaryThresholdInDecibels; var midBandSpectrogram = MatrixTools.Submatrix(deciBelSpectrogram, 0, lowerBinBound, deciBelSpectrogram.GetLength(0) - 1, middleBinBound); var clusterInfo = SpectralClustering.ClusterTheSpectra(midBandSpectrogram, lowerBinBound, middleBinBound, binaryThreshold); // Store two summary index values from cluster info summaryIndices.ClusterCount = clusterInfo.ClusterCount; summaryIndices.ThreeGramCount = clusterInfo.TriGramUniqueCount; // As of May 2017, no longer store clustering results superimposed on spectrogram. // If you want to see this, then call the TEST methods in class SpectralClustering.cs. // ####################################################################################################################################################### // vii: set up other info to return var freqPeaks = SpectralPeakTracks.ConvertSpectralPeaksToNormalisedArray(deciBelSpectrogram); var scores = new List <Plot> { new Plot("Decibels", DataTools.normalise(dBEnvelopeSansNoise), ActivityAndCover.DefaultActivityThresholdDb), new Plot("Active Frames", DataTools.Bool2Binary(activity.ActiveFrames), 0.0), new Plot("Max Frequency", freqPeaks, 0.0), // relative location of freq maxima in spectra }; result.Hits = sptInfo.Peaks; result.TrackScores = scores; return(result); } // end Calculation of Summary and Spectral Indices