} //GetExcludeRules() //filter events using the exclude rules - just changes the event's normalised score public static AcousticEvent FilterEvent(AcousticEvent ae, List <string[]> rules) { //loop through exclusion rules - DO NOT DELETE events - set score to zero so can check later what is happening. foreach (string[] rule in rules) { string feature = rule[0]; string op = rule[1]; double value = double.Parse(rule[2]); //if (feature == key_BANDWIDTH_SCORE) //{ // if ((op == "LT" && ae.kiwi_bandWidthScore < value) || (op == "GT" && ae.kiwi_bandWidthScore > value)) // { // ae.kiwi_bandWidthScore = 0.0; // ae.ScoreNormalised = 0.0; // return ae; // } //} //else // end if key_BANDWIDTH_SCORE // if (feature == key_INTENSITY_SCORE) // { // if ((op == "LT" && ae.kiwi_intensityScore < value) || (op == "GT" && ae.kiwi_intensityScore > value)) // { // ae.ScoreNormalised = 0.0; // return ae; // } // } // end if key_INTENSITY_SCORE } return(ae); }
/// <summary> /// /// </summary> /// <param name="ae">an acoustic event</param> /// <param name="dbArray">The sequence of frame dB over the event</param> /// <returns></returns> public static System.Tuple <double, double> KiwiPeakAnalysis(AcousticEvent ae, double[] dbArray) { //dbArray = DataTools.filterMovingAverage(dbArray, 3); bool[] peaks = DataTools.GetPeaks(dbArray); //locate the peaks double[] peakValues = new double[dbArray.Length]; for (int i = 0; i < dbArray.Length; i++) { if (peaks[i]) { peakValues[i] = dbArray[i]; } } //take the top N peaks int N = 5; double[] topNValues = new double[N]; for (int p = 0; p < N; p++) { int maxID = DataTools.GetMaxIndex(peakValues); topNValues[p] = peakValues[maxID]; peakValues[maxID] = 0.0; } //PROCESS PEAK DECIBELS double avPeakDB, sdPeakDB; NormalDist.AverageAndSD(topNValues, out avPeakDB, out sdPeakDB); return(System.Tuple.Create(avPeakDB, sdPeakDB)); }
public static List <AcousticEvent> ReadGroundParrotTemplateAsList(BaseSonogram sonogram) { var timeScale = sonogram.FrameStep; var hzScale = (int)sonogram.FBinWidth; int rows = GroundParrotTemplate1.GetLength(0); double timeOffset = GroundParrotTemplate1[0, 0]; var gpTemplate = new List <AcousticEvent>(); for (int r = 0; r < rows; r++) { int t1 = (int)Math.Round((GroundParrotTemplate1[r, 0] - timeOffset) / timeScale); int t2 = (int)Math.Round((GroundParrotTemplate1[r, 1] - timeOffset) / timeScale); int f2 = (int)Math.Round(GroundParrotTemplate1[r, 2] / hzScale); int f1 = (int)Math.Round(GroundParrotTemplate1[r, 3] / hzScale); Oblong o = new Oblong(t1, f1, t2, f2); var ae = AcousticEvent.InitializeAcousticEvent( TimeSpan.Zero, o, sonogram.NyquistFrequency, sonogram.Configuration.FreqBinCount, sonogram.FrameDuration, sonogram.FrameStep, sonogram.FrameCount); gpTemplate.Add(ae); } return(gpTemplate); }
public static double[,] ExtractFreqSubband(double[,] m, int minHz, int maxHz, bool doMelscale, int binCount, double binWidth) { int c1; int c2; AcousticEvent.Freq2BinIDs(doMelscale, minHz, maxHz, binCount, binWidth, out c1, out c2); return(DataTools.Submatrix(m, 0, c1, m.GetLength(0) - 1, c2)); }
static CsvTests() { // pump static class initializers - it seems when running multiple tests that sometimes // these classes are not discovered. _ = new AcousticEvent(); _ = new ImportedEvent(); }
/// <summary> /// FINDS OSCILLATIONS IN A SONOGRAM /// But first it segments the sonogram based on acoustic energy in freq band of interest. /// </summary> /// <param name="sonogram">sonogram derived from the recording.</param> /// <param name="minHz">min bound freq band to search.</param> /// <param name="maxHz">max bound freq band to search.</param> /// <param name="dctDuration">duration of DCT in seconds.</param> /// <param name="dctThreshold">minimum amplitude of DCT. </param> /// <param name="minOscilFreq">ignore oscillation frequencies below this threshold.</param> /// <param name="maxOscilFreq">ignore oscillation frequencies greater than this. </param> /// <param name="scoreThreshold">used for FP/FN.</param> /// <param name="minDuration">ignore hits whose duration is shorter than this.</param> /// <param name="maxDuration">ignore hits whose duration is longer than this.</param> /// <param name="scores">return an array of scores over the entire recording.</param> /// <param name="events">return a list of acoustic events.</param> /// <param name="hits">a matrix that show where there is an oscillation of sufficient amplitude in the correct range. /// Values in the matrix are the oscillation rate. i.e. if OR = 2.0 = 2 oscillations per second. </param> public static void Execute(SpectrogramStandard sonogram, bool doSegmentation, int minHz, int maxHz, double dctDuration, double dctThreshold, bool normaliseDCT, int minOscilFreq, int maxOscilFreq, double scoreThreshold, double minDuration, double maxDuration, out double[] scores, out List <AcousticEvent> events, out double[,] hits, out double[] intensity, out TimeSpan totalTime) { DateTime startTime1 = DateTime.Now; //EXTRACT SEGMENTATIOn EVENTS //DO SEGMENTATION double smoothWindow = 1 / (double)minOscilFreq; //window = max oscillation period Log.WriteLine(" Segmentation smoothing window = {0:f2} seconds", smoothWindow); double thresholdSD = 0.1; //Set threshold to 1/5th of a standard deviation of the background noise. maxDuration = double.MaxValue; //Do not constrain maximum length of events. var tuple = AcousticEvent.GetSegmentationEvents(sonogram, doSegmentation, TimeSpan.Zero, minHz, maxHz, smoothWindow, thresholdSD, minDuration, maxDuration); var segmentEvents = tuple.Item1; intensity = tuple.Item5; Log.WriteLine("Number of segments={0}", segmentEvents.Count); TimeSpan span1 = DateTime.Now.Subtract(startTime1); Log.WriteLine(" SEGMENTATION COMP TIME = " + span1.TotalMilliseconds.ToString() + "ms"); DateTime startTime2 = DateTime.Now; //DETECT OSCILLATIONS hits = DetectOscillationsInSonogram(sonogram, minHz, maxHz, dctDuration, dctThreshold, normaliseDCT, minOscilFreq, maxOscilFreq, segmentEvents); hits = RemoveIsolatedOscillations(hits); //EXTRACT SCORES AND ACOUSTIC EVENTS scores = GetOscillationScores(hits, minHz, maxHz, sonogram.FBinWidth); double[] oscFreq = GetOscillationFrequency(hits, minHz, maxHz, sonogram.FBinWidth); events = ConvertODScores2Events( scores, oscFreq, minHz, maxHz, sonogram.FramesPerSecond, sonogram.FBinWidth, sonogram.Configuration.FreqBinCount, scoreThreshold, minDuration, maxDuration, sonogram.Configuration.SourceFName, TimeSpan.Zero); //events = segmentEvents; //#################################### to see segment events in output image. DateTime endTime2 = DateTime.Now; TimeSpan span2 = endTime2.Subtract(startTime2); Log.WriteLine(" TOTAL COMP TIME = " + span2.ToString() + "s"); totalTime = endTime2.Subtract(startTime1); }
public void TestEventMerging() { // make a list of events var events = new List <AcousticEvent>(); double maxPossibleScore = 10.0; var event1 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 1.0, eventDuration: 5.0, minFreq: 1000, maxFreq: 8000) { Name = "Event1", Score = 1.0, ScoreNormalised = 1.0 / maxPossibleScore, }; events.Add(event1); var event2 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 4.5, eventDuration: 2.0, minFreq: 1500, maxFreq: 6000) { Name = "Event2", Score = 5.0, ScoreNormalised = 5.0 / maxPossibleScore, }; events.Add(event2); var event3 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 7.0, eventDuration: 2.0, minFreq: 1000, maxFreq: 8000) { Name = "Event3", Score = 9.0, ScoreNormalised = 9.0 / maxPossibleScore, }; events.Add(event3); // combine adjacent acoustic events events = AcousticEvent.CombineOverlappingEvents(events: events, segmentStartOffset: TimeSpan.Zero); Assert.AreEqual(2, events.Count); Assert.AreEqual(1.0, events[0].EventStartSeconds, 1E-4); Assert.AreEqual(6.5, events[0].EventEndSeconds, 1E-4); Assert.AreEqual(7.0, events[1].EventStartSeconds, 1E-4); Assert.AreEqual(9.0, events[1].EventEndSeconds, 1E-4); Assert.AreEqual(1000, events[0].LowFrequencyHertz); Assert.AreEqual(8000, events[0].HighFrequencyHertz); Assert.AreEqual(1000, events[1].LowFrequencyHertz); Assert.AreEqual(8000, events[1].HighFrequencyHertz); Assert.AreEqual(5.0, events[0].Score, 1E-4); Assert.AreEqual(9.0, events[1].Score, 1E-4); Assert.AreEqual(0.5, events[0].ScoreNormalised, 1E-4); Assert.AreEqual(0.9, events[1].ScoreNormalised, 1E-4); }
public static Tuple <BaseSonogram, AcousticEvent, double[, ], double[], double[, ]> Execute_Extraction( AudioRecording recording, double eventStart, double eventEnd, int minHz, int maxHz, double frameOverlap, double backgroundThreshold, TimeSpan segmentStartOffset) { //ii: MAKE SONOGRAM SonogramConfig sonoConfig = new SonogramConfig(); //default values config sonoConfig.SourceFName = recording.BaseName; //sonoConfig.WindowSize = windowSize; sonoConfig.WindowOverlap = frameOverlap; BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); Log.WriteLine("Frames: Size={0}, Count={1}, Duration={2:f1}ms, Overlap={5:f2}%, Offset={3:f1}ms, Frames/s={4:f1}", sonogram.Configuration.WindowSize, sonogram.FrameCount, (sonogram.FrameDuration * 1000), (sonogram.FrameStep * 1000), sonogram.FramesPerSecond, frameOverlap); int binCount = (int)(maxHz / sonogram.FBinWidth) - (int)(minHz / sonogram.FBinWidth) + 1; Log.WriteIfVerbose("Freq band: {0} Hz - {1} Hz. (Freq bin count = {2})", minHz, maxHz, binCount); //calculate the modal noise profile double SD_COUNT = 0.0; // number of noise standard deviations used to calculate noise threshold NoiseProfile profile = NoiseProfile.CalculateModalNoiseProfile(sonogram.Data, SD_COUNT); //calculate modal noise profile double[] modalNoise = DataTools.filterMovingAverage(profile.NoiseMode, 7); //smooth the noise profile //extract modal noise values of the required event double[] noiseSubband = SpectrogramTools.ExtractModalNoiseSubband(modalNoise, minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth); //extract data values of the required event double[,] target = SpectrogramTools.ExtractEvent(sonogram.Data, eventStart, eventEnd, sonogram.FrameStep, minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth); // create acoustic event with defined boundaries AcousticEvent ae = new AcousticEvent(segmentStartOffset, eventStart, eventEnd - eventStart, minHz, maxHz); ae.SetTimeAndFreqScales(sonogram.FramesPerSecond, sonogram.FBinWidth); //truncate noise sonogram.Data = SNR.TruncateBgNoiseFromSpectrogram(sonogram.Data, modalNoise); sonogram.Data = SNR.RemoveNeighbourhoodBackgroundNoise(sonogram.Data, backgroundThreshold); double[,] targetMinusNoise = SpectrogramTools.ExtractEvent(sonogram.Data, eventStart, eventEnd, sonogram.FrameStep, minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth); return(Tuple.Create(sonogram, ae, target, noiseSubband, targetMinusNoise)); }
/// <summary> /// Extracts an acoustic event from a sonogram given the location of a user defined rectangular marquee. /// NOTE: Nyquist value is used ONLY if using mel scale. /// </summary> /// <param name="m">the sonogram data as matrix of reals</param> /// <param name="start">start time in seconds</param> /// <param name="end">end time in seconds</param> /// <param name="frameOffset">the time scale: i.e. the duration in seconds of each frame</param> /// <param name="minHz">lower freq bound of the event</param> /// <param name="maxHz">upper freq bound of the event</param> /// <param name="doMelscale">informs whether the sonogram data is linear or mel scale</param> /// <param name="nyquist">full freq range 0-Nyquist</param> /// <param name="binWidth">the frequency scale i.e. herz per bin width - assumes linear scale</param> /// <returns></returns> public static double[,] ExtractEvent(double[,] m, double start, double end, double frameOffset, int minHz, int maxHz, bool doMelscale, int nyquist, double binWidth) { int r1; int r2; AcousticEvent.Time2RowIDs(start, end - start, frameOffset, out r1, out r2); int c1; int c2; AcousticEvent.Freq2BinIDs(doMelscale, minHz, maxHz, nyquist, binWidth, out c1, out c2); return(DataTools.Submatrix(m, r1, c1, r2, c2)); }
static CsvTests() { // pump static class initializers - it seems when running multiple tests that sometimes // these classes are not discovered. AcousticEvent aev = new AcousticEvent(); ImportedEvent iev = new ImportedEvent(); var configuration = Csv.DefaultConfiguration; IDictionary csvMaps = (IDictionary)configuration.Maps.GetFieldValue("data"); Debug.WriteLine("initializing static types" + aev + iev + csvMaps.Count); }
public static SpectralEvent ConvertAcousticEventToSpectralEvent(this AcousticEvent ae) { var segmentOffset = TimeSpan.FromSeconds(ae.SegmentStartSeconds); var eventStart = ae.EventStartSeconds; var eventEnd = ae.EventEndSeconds; return(new SpectralEvent(segmentStartOffset: segmentOffset, eventStartRecordingRelative: eventStart, eventEndRecordingRelative: eventEnd, ae.LowFrequencyHertz, ae.HighFrequencyHertz) { Name = ae.Name, //SegmentStartSeconds = ae.SegmentStartSeconds, SegmentDurationSeconds = ae.SegmentDurationSeconds, }); }
public static double[] ExtractModalNoiseSubband(double[] modalNoise, int minHz, int maxHz, bool doMelScale, int nyquist, double binWidth) { //extract subband modal noise profile AcousticEvent.ConvertHertzToFrequencyBin(doMelScale, minHz, maxHz, nyquist, binWidth, out var c1, out var c2); int subbandCount = c2 - c1 + 1; var subband = new double[subbandCount]; for (int i = 0; i < subbandCount; i++) { subband[i] = modalNoise[c1 + i]; } return(subband); }
public static AcousticEvent ConvertSpectralEventToAcousticEvent(this SpectralEvent se) { var segmentStartOffset = TimeSpan.FromSeconds(se.SegmentStartSeconds); double startTime = se.EventStartSeconds; double duration = se.EventDurationSeconds; double minHz = se.HighFrequencyHertz; double maxHz = se.HighFrequencyHertz; var ae = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz) { Name = se.Name, SegmentDurationSeconds = se.SegmentDurationSeconds, }; return(ae); }
/// <summary> /// Detect using EPR. /// </summary> /// <param name="wavFilePath"> /// The wav file path. /// </param> /// <param name="eprNormalisedMinScore"> /// The epr Normalised Min Score. /// </param> /// <returns> /// Tuple containing base Sonogram and list of acoustic events. /// </returns> public static Tuple <BaseSonogram, List <AcousticEvent> > Detect(FileInfo wavFilePath, Aed.AedConfiguration aedConfiguration, double eprNormalisedMinScore, TimeSpan segmentStartOffset) { Tuple <EventCommon[], AudioRecording, BaseSonogram> aed = Aed.Detect(wavFilePath, aedConfiguration, segmentStartOffset); var events = aed.Item1; var newEvents = new List <AcousticEvent>(); foreach (var be in events) { newEvents.Add(EventConverters.ConvertSpectralEventToAcousticEvent((SpectralEvent)be)); } var aeEvents = newEvents.Select(ae => Util.fcornersToRect(ae.TimeStart, ae.TimeEnd, ae.HighFrequencyHertz, ae.LowFrequencyHertz)).ToList(); Log.Debug("EPR start"); var eprRects = EventPatternRecog.DetectGroundParrots(aeEvents, eprNormalisedMinScore); Log.Debug("EPR finished"); var sonogram = aed.Item3; SonogramConfig config = sonogram.Configuration; double framesPerSec = 1 / config.GetFrameOffset(); // Surely this should go somewhere else double freqBinWidth = config.NyquistFreq / (double)config.FreqBinCount; // TODO this is common with AED var eprEvents = new List <AcousticEvent>(); foreach (var rectScore in eprRects) { var ae = new AcousticEvent( segmentStartOffset, rectScore.Item1.Left, rectScore.Item1.Right - rectScore.Item1.Left, rectScore.Item1.Bottom, rectScore.Item1.Top); ae.SetTimeAndFreqScales(framesPerSec, freqBinWidth); ae.SetTimeAndFreqScales(sonogram.NyquistFrequency, sonogram.Configuration.WindowSize, 0); ae.SetScores(rectScore.Item2, 0, 1); ae.BorderColour = aedConfiguration.AedEventColor; ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; ae.SegmentDurationSeconds = aed.Item2.Duration.TotalSeconds; eprEvents.Add(ae); } return(Tuple.Create(sonogram, eprEvents)); }
public void TestSonogramWithEventsOverlay() { // make a substitute sonogram image var substituteSonogram = Drawing.NewImage(100, 256, Color.Black); // make a list of events var framesPerSecond = 10.0; var freqBinWidth = 43.0664; double maxPossibleScore = 10.0; var events = new List <AcousticEvent>(); var event1 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 1.0, eventDuration: 5.0, minFreq: 1000, maxFreq: 8000) { Score = 10.0, Name = "Event1", ScoreNormalised = 10.0 / maxPossibleScore, }; events.Add(event1); var event2 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 7.0, eventDuration: 2.0, minFreq: 1000, maxFreq: 8000) { Score = 1.0, Name = "Event2", ScoreNormalised = 1.0 / maxPossibleScore, }; events.Add(event2); // now add events into the spectrogram image. // set color for the events foreach (AcousticEvent ev in events) { // because we are testing placement of box not text. ev.Name = string.Empty; ev.BorderColour = AcousticEvent.DefaultBorderColor; ev.ScoreColour = AcousticEvent.DefaultScoreColor; ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, 256); } this.ActualImage = substituteSonogram; // BUG: this asset is faulty. See https://github.com/QutEcoacoustics/audio-analysis/issues/300#issuecomment-601537263 this.ExpectedImage = Image.Load <Rgb24>( PathHelper.ResolveAssetPath("AcousticEventTests_SuperimposeEventsOnImage.png")); this.AssertImagesEqual(); }
public static void DrawSonogram(BaseSonogram sonogram, string path, AcousticEvent ae) { Log.WriteLine("# Start to draw image of sonogram."); bool doHighlightSubband = false; bool add1kHzLines = true; using (Image img = sonogram.GetImage(doHighlightSubband, add1kHzLines)) using (Image_MultiTrack image = new Image_MultiTrack(img)) { //img.Save(@"C:\SensorNetworks\WavFiles\temp1\testimage1.jpg", System.Drawing.Imaging.ImageFormat.Jpeg); image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond)); image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); var aes = new List <AcousticEvent>(); aes.Add(ae); image.AddEvents(aes, sonogram.NyquistFrequency, sonogram.Configuration.FreqBinCount, sonogram.FramesPerSecond); image.Save(path); } } //end DrawSonogram
public void TestBaseTypesAreSerializedAsEnumerableAcousticEvent() { var exampleEvent = new AcousticEvent(100.Seconds(), 15, 4, 100, 3000); var exampleEvent2 = new AcousticEvent(100.Seconds(), 15, 4, 100, 3000); AcousticEvent[] childArray = { exampleEvent, exampleEvent2 }; EventBase[] baseArray = { exampleEvent, exampleEvent2 }; Csv.WriteToCsv(this.testFile, childArray); var childText = File.ReadAllText(this.testFile.FullName); Csv.WriteToCsv(this.testFile, baseArray); var baseText = File.ReadAllText(this.testFile.FullName); Assert.AreNotEqual(childText, baseText); }
public void TestGetEventsAroundMaxima() { //string abbreviatedSpeciesName = "Pteropus"; string speciesName = "Pteropus species"; int minHz = 800; int maxHz = 8000; var minTimeSpan = TimeSpan.FromSeconds(0.15); var maxTimeSpan = TimeSpan.FromSeconds(0.8); double decibelThreshold = 9.0; TimeSpan segmentStartOffset = TimeSpan.Zero; var decibelArray = SNR.CalculateFreqBandAvIntensity(this.sonogram.Data, minHz, maxHz, this.sonogram.NyquistFrequency); // prepare plots double intensityNormalisationMax = 3 * decibelThreshold; var eventThreshold = decibelThreshold / intensityNormalisationMax; var normalisedIntensityArray = DataTools.NormaliseInZeroOne(decibelArray, 0, intensityNormalisationMax); var plot = new Plot(speciesName + " Territory", normalisedIntensityArray, eventThreshold); var plots = new List <Plot> { plot }; //iii: CONVERT decibel SCORES TO ACOUSTIC EVENTS var acousticEvents = AcousticEvent.GetEventsAroundMaxima( decibelArray, segmentStartOffset, minHz, maxHz, decibelThreshold, minTimeSpan, maxTimeSpan, this.sonogram.FramesPerSecond, this.sonogram.FBinWidth); Assert.AreEqual(10, acousticEvents.Count); Assert.AreEqual(new Rectangle(19, 1751, 168, 27), acousticEvents[0].GetEventAsRectangle()); Assert.AreEqual(new Rectangle(19, 1840, 168, 10), acousticEvents[2].GetEventAsRectangle()); Assert.AreEqual(new Rectangle(19, 1961, 168, 31), acousticEvents[5].GetEventAsRectangle()); Assert.AreEqual(new Rectangle(19, 2294, 168, 17), acousticEvents[7].GetEventAsRectangle()); Assert.AreEqual(new Rectangle(19, 2504, 168, 7), acousticEvents[9].GetEventAsRectangle()); //Assert.AreEqual(28.Seconds() + segmentOffset, stats.ResultStartSeconds.Seconds()); }
public static double CalculateAverageEventScore(AcousticEvent ae, double[] scoreArray) { int start = ae.Oblong.RowTop; int end = ae.Oblong.RowBottom; if (end > scoreArray.Length) { end = scoreArray.Length - 1; } int length = end - start + 1; double sum = 0.0; for (int i = start; i <= end; i++) { sum += scoreArray[i]; } return(sum / length); }
} // FELTWithSprTemplate() /// <summary> /// Edits the passed events to adjust start and end locations to position of maximum score /// Also correct the scores because we want the max match for a template /// Also correct the time scale /// </summary> /// <param name="matchEvents"></param> /// <param name="callName"></param> /// <param name="templateDuration"></param> /// <param name="sonogramDuration"></param> public static void AdjustEventLocation(List <AcousticEvent> matchEvents, string callName, double templateDuration, double sonogramDuration) { Log.WriteLine("# ADJUST EVENT LOCATIONS"); Log.WriteLine("# Event: " + callName); foreach (AcousticEvent ae in matchEvents) { Log.WriteLine("# Old event frame= {0} to {1}.", ae.Oblong.RowTop, ae.Oblong.RowBottom); ae.Name = callName; ae.TimeStart = ae.Score_TimeOfMaxInEvent; ae.TimeEnd = ae.TimeStart + templateDuration; if (ae.TimeEnd > sonogramDuration) { ae.TimeEnd = sonogramDuration; // check for overflow. } ae.Oblong = AcousticEvent.ConvertEvent2Oblong(ae); ae.Score = ae.Score_MaxInEvent; ae.ScoreNormalised = ae.Score / ae.Score_MaxPossible; // normalised to the user supplied threshold Log.WriteLine("# New event time = {0:f2} to {1:f2}.", ae.TimeStart, ae.TimeEnd); Log.WriteLine("# New event frame= {0} to {1}.", ae.Oblong.RowTop, ae.Oblong.RowBottom); } }
/// <summary> /// /// </summary> /// <param name="wavPath"></param> /// <param name="minHz"></param> /// <param name="maxHz"></param> /// <param name="frameOverlap"></param> /// <param name="threshold"></param> /// <param name="minDuration">used for smoothing intensity as well as for removing short events</param> /// <param name="maxDuration"></param> /// <returns></returns> public static Tuple <BaseSonogram, List <AcousticEvent>, double, double, double, double[]> Execute_Segmentation(FileInfo wavPath, int minHz, int maxHz, double frameOverlap, double smoothWindow, double thresholdSD, double minDuration, double maxDuration) { //i: GET RECORDING AudioRecording recording = new AudioRecording(wavPath.FullName); //ii: MAKE SONOGRAM Log.WriteLine("# Start sonogram."); SonogramConfig sonoConfig = new SonogramConfig(); //default values config sonoConfig.WindowOverlap = frameOverlap; sonoConfig.SourceFName = recording.BaseName; BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); //iii: DETECT SEGMENTS Log.WriteLine("# Start event detection"); var tuple = AcousticEvent.GetSegmentationEvents((SpectrogramStandard)sonogram, TimeSpan.Zero, minHz, maxHz, smoothWindow, thresholdSD, minDuration, maxDuration); var tuple2 = Tuple.Create(sonogram, tuple.Item1, tuple.Item2, tuple.Item3, tuple.Item4, tuple.Item5); return(tuple2); }//end Execute_Segmentation
public static List <AcousticEvent> ConvertTracks2Events( List <SpectralTrack> tracks, TimeSpan segmentStartOffset) /*, double framesPerSecond, double herzPerBin*/ { if (tracks == null) { return(null); } var list = new List <AcousticEvent>(); if (tracks.Count == 0) { return(list); } foreach (SpectralTrack track in tracks) { double startTime = track.StartFrame / track.framesPerSecond; int frameDuration = track.EndFrame - track.StartFrame + 1; double duration = frameDuration / track.framesPerSecond; double minFreq = track.herzPerBin * (track.AverageBin - 1); double maxFreq = track.herzPerBin * (track.AverageBin + 1); AcousticEvent ae = new AcousticEvent(segmentStartOffset, startTime, duration, minFreq, maxFreq); ae.SetTimeAndFreqScales(track.framesPerSecond, track.herzPerBin); ae.Name = ""; ae.BorderColour = Color.Blue; ae.DominantFreq = track.AverageBin * track.herzPerBin; ae.Periodicity = track.avPeriodicity; ae.Score = track.avPeriodicityScore; list.Add(ae); } return(list); }
public static List <AcousticEvent> PruneOverlappingEvents(List <AcousticEvent> events, int percentOverlap) { double thresholdOverlap = percentOverlap / 100.0; int count = events.Count; for (int i = 0; i < count - 1; i++) { for (int j = i + 1; j < count; j++) { if (AcousticEvent.EventFractionalOverlap(events[i], events[j]) > thresholdOverlap) { //determine the event with highest score if (events[i].Score >= events[j].Score) { events[j].Name = null; } else { events[i].Name = null; } } } } List <AcousticEvent> pruned = new List <AcousticEvent>(); foreach (AcousticEvent ae in events) { if (ae.Name != null) { pruned.Add(ae); } } return(pruned); }
public void TestAcousticEventClassMap() { var ae = new AcousticEvent(); var result = new StringBuilder(); using (var str = new StringWriter(result)) { var writer = new CsvWriter(str, Csv.DefaultConfiguration); writer.WriteRecords(records: new[] { ae }); } var actual = result.ToString(); foreach (var property in AcousticEvent.AcousticEventClassMap.IgnoredProperties.Except(AcousticEvent.AcousticEventClassMap.RemappedProperties)) { Assert.IsFalse( actual.Contains(property, StringComparison.InvariantCultureIgnoreCase), $"output CSV should not contain text '{property}'.{Environment.NewLine}Actual: {actual}"); } StringAssert.Contains(actual, "EventEndSeconds"); }
public static double[,] ExtractFreqSubband(double[,] m, int minHz, int maxHz, bool doMelscale, int binCount, double binWidth) { AcousticEvent.ConvertHertzToFrequencyBin(doMelscale, minHz, maxHz, binCount, binWidth, out var c1, out var c2); return(DataTools.Submatrix(m, 0, c1, m.GetLength(0) - 1, c2)); }
/// <summary> /// THE KEY ANALYSIS METHOD /// </summary> /// <param name="recording"> /// The segment Of Source File. /// </param> /// <param name="configDict"> /// The config Dict. /// </param> /// <param name="value"></param> /// <returns> /// The <see cref="LimnodynastesConvexResults"/>. /// </returns> internal static LimnodynastesConvexResults Analysis( Dictionary <string, double[, ]> dictionaryOfHiResSpectralIndices, AudioRecording recording, Dictionary <string, string> configDict, AnalysisSettings analysisSettings, SegmentSettingsBase segmentSettings) { // for Limnodynastes convex, in the D.Stewart CD, there are peaks close to: //1. 1950 Hz //2. 1460 hz //3. 970 hz These are 490 Hz apart. // for Limnodynastes convex, in the JCU recording, there are peaks close to: //1. 1780 Hz //2. 1330 hz //3. 880 hz These are 450 Hz apart. // So strategy is to look for three peaks separated by same amount and in the vicinity of the above, // starting with highest power (the top peak) and working down to lowest power (bottom peak). var outputDir = segmentSettings.SegmentOutputDirectory; TimeSpan segmentStartOffset = segmentSettings.SegmentStartOffset; //KeyValuePair<string, double[,]> kvp = dictionaryOfHiResSpectralIndices.First(); var spg = dictionaryOfHiResSpectralIndices["RHZ"]; int rhzRowCount = spg.GetLength(0); int rhzColCount = spg.GetLength(1); int sampleRate = recording.SampleRate; double herzPerBin = sampleRate / 2 / (double)rhzRowCount; double scoreThreshold = (double?)double.Parse(configDict["EventThreshold"]) ?? 3.0; int minimumFrequency = (int?)int.Parse(configDict["MinHz"]) ?? 850; int dominantFrequency = (int?)int.Parse(configDict["DominantFrequency"]) ?? 1850; // # The Limnodynastes call has three major peaks. The dominant peak is at 1850 or as set above. // # The second and third peak are at equal gaps below. DominantFreq-gap and DominantFreq-(2*gap); // # Set the gap in the Config file. Should typically be in range 880 to 970 int peakGapInHerz = (int?)int.Parse(configDict["PeakGap"]) ?? 470; int F1AndF2Gap = (int)Math.Round(peakGapInHerz / herzPerBin); //int F1AndF2Gap = 10; // 10 = number of freq bins int F1AndF3Gap = 2 * F1AndF2Gap; //int F1AndF3Gap = 20; int hzBuffer = 250; int bottomBin = 5; int dominantBin = (int)Math.Round(dominantFrequency / herzPerBin); int binBuffer = (int)Math.Round(hzBuffer / herzPerBin);; int dominantBinMin = dominantBin - binBuffer; int dominantBinMax = dominantBin + binBuffer; // freqBin + rowID = binCount - 1; // therefore: rowID = binCount - freqBin - 1; int minRowID = rhzRowCount - dominantBinMax - 1; int maxRowID = rhzRowCount - dominantBinMin - 1; int bottomRow = rhzRowCount - bottomBin - 1; var list = new List <Point>(); // loop through all spectra/columns of the hi-res spectrogram. for (int c = 1; c < rhzColCount - 1; c++) { double maxAmplitude = -double.MaxValue; int idOfRowWithMaxAmplitude = 0; for (int r = minRowID; r <= bottomRow; r++) { if (spg[r, c] > maxAmplitude) { maxAmplitude = spg[r, c]; idOfRowWithMaxAmplitude = r; } } if (idOfRowWithMaxAmplitude < minRowID) { continue; } if (idOfRowWithMaxAmplitude > maxRowID) { continue; } // want a spectral peak. if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c - 1]) { continue; } if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c + 1]) { continue; } // peak should exceed thresold amplitude if (spg[idOfRowWithMaxAmplitude, c] < 3.0) { continue; } // convert row ID to freq bin ID int freqBinID = rhzRowCount - idOfRowWithMaxAmplitude - 1; list.Add(new Point(c, freqBinID)); // we now have a list of potential hits for LimCon. This needs to be filtered. // Console.WriteLine("Col {0}, Bin {1} ", c, freqBinID); } // DEBUG ONLY // ################################ TEMPORARY ################################ // superimpose point on RHZ HiRes spectrogram for debug purposes bool drawOnHiResSpectrogram = true; //string filePath = @"G:\SensorNetworks\Output\Frogs\TestOfHiResIndices-2016July\Test\Towsey.HiResIndices\SpectrogramImages\3mile_creek_dam_-_Herveys_Range_1076_248366_20130305_001700_30_0min.CombinedGreyScale.png"; var fileName = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name); string filePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScale.png"; var debugImage = new FileInfo(filePath); if (!debugImage.Exists) { drawOnHiResSpectrogram = false; } if (drawOnHiResSpectrogram) { // put red dot where max is Bitmap bmp = new Bitmap(filePath); foreach (Point point in list) { bmp.SetPixel(point.X + 70, 1911 - point.Y, Color.Red); } // mark off every tenth frequency bin for (int r = 0; r < 26; r++) { bmp.SetPixel(68, 1911 - (r * 10), Color.Blue); bmp.SetPixel(69, 1911 - (r * 10), Color.Blue); } // mark off upper bound and lower frequency bound bmp.SetPixel(69, 1911 - dominantBinMin, Color.Lime); bmp.SetPixel(69, 1911 - dominantBinMax, Color.Lime); //bmp.SetPixel(69, 1911 - maxRowID, Color.Lime); string opFilePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScaleAnnotated.png"; bmp.Save(opFilePath); } // END DEBUG ################################ TEMPORARY ################################ // now construct the standard decibel spectrogram WITHOUT noise removal, and look for LimConvex // get frame parameters for the analysis double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1); int frameSize = rhzRowCount * 2; int frameStep = frameSize; // this default = zero overlap double frameDurationInSeconds = frameSize / (double)sampleRate; double frameStepInSeconds = frameStep / (double)sampleRate; double framesPerSec = 1 / frameStepInSeconds; //var dspOutput = DSP_Frames.ExtractEnvelopeAndFFTs(recording, frameSize, frameStep); //// Generate deciBel spectrogram //double[,] deciBelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput.amplitudeSpectrogram, dspOutput.WindowPower, sampleRate, epsilon); // i: Init SONOGRAM config var sonoConfig = new SonogramConfig { SourceFName = recording.BaseName, WindowSize = frameSize, WindowOverlap = 0.0, NoiseReductionType = NoiseReductionType.None, }; // init sonogram BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // remove the DC row of the spectrogram sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.Data.GetLength(0) - 1, sonogram.Data.GetLength(1) - 1); //scores.Add(new Plot("Decibels", DataTools.NormaliseMatrixValues(dBArray), ActivityAndCover.DefaultActivityThresholdDb)); //scores.Add(new Plot("Active Frames", DataTools.Bool2Binary(activity.activeFrames), 0.0)); // convert spectral peaks to frequency //var tuple_DecibelPeaks = SpectrogramTools.HistogramOfSpectralPeaks(deciBelSpectrogram); //int[] peaksBins = tuple_DecibelPeaks.Item2; //double[] freqPeaks = new double[peaksBins.Length]; //int binCount = sonogram.Data.GetLength(1); //for (int i = 1; i < peaksBins.Length; i++) freqPeaks[i] = (lowerBinBound + peaksBins[i]) / (double)nyquistBin; //scores.Add(new Plot("Max Frequency", freqPeaks, 0.0)); // location of peaks for spectral images // create new list of LimCon hits in the standard spectrogram. double timeSpanOfFrameInSeconds = frameSize / (double)sampleRate; var newList = new List <int[]>(); int lastFrameID = sonogram.Data.GetLength(0) - 1; int lastBinID = sonogram.Data.GetLength(1) - 1; foreach (Point point in list) { double secondsFromStartOfSegment = (point.X * 0.1) + 0.05; // convert point.Y to center of time-block. int framesFromStartOfSegment = (int)Math.Round(secondsFromStartOfSegment / timeSpanOfFrameInSeconds); // location of max point is uncertain, so search in neighbourhood. // NOTE: sonogram.data matrix is time*freqBin double maxValue = -double.MaxValue; int idOfTMax = framesFromStartOfSegment; int idOfFMax = point.Y; for (int deltaT = -4; deltaT <= 4; deltaT++) { for (int deltaF = -1; deltaF <= 1; deltaF++) { int newT = framesFromStartOfSegment + deltaT; if (newT < 0) { newT = 0; } else if (newT > lastFrameID) { newT = lastFrameID; } double value = sonogram.Data[newT, point.Y + deltaF]; if (value > maxValue) { maxValue = value; idOfTMax = framesFromStartOfSegment + deltaT; idOfFMax = point.Y + deltaF; } } } // newList.Add(new Point(frameSpan, point.Y)); int[] array = new int[2]; array[0] = idOfTMax; array[1] = idOfFMax; newList.Add(array); } // Now obtain more of spectrogram to see if have peaks at two other places characteristic of Limnodynastes convex. // In the D.Stewart CD, there are peaks close to: //1. 1950 Hz //2. 1460 hz //3. 970 hz These are 490 Hz apart. // For Limnodynastes convex, in the JCU recording, there are peaks close to: //1. 1780 Hz //2. 1330 hz //3. 880 hz These are 450 Hz apart. // So strategy is to look for three peaks separated by same amount and in the vicinity of the above, // starting with highest power (the top peak) and working down to lowest power (bottom peak). //We have found top/highest peak - now find the other two. int secondDominantFrequency = 1380; int secondDominantBin = (int)Math.Round(secondDominantFrequency / herzPerBin); int thirdDominantFrequency = 900; int thirdDominantBin = (int)Math.Round(thirdDominantFrequency / herzPerBin); var acousticEvents = new List <AcousticEvent>(); int Tbuffer = 2; // First extract a sub-matrix. foreach (int[] array in newList) { // NOTE: sonogram.data matrix is time*freqBin int Tframe = array[0]; int F1bin = array[1]; double[,] subMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - Tbuffer, 0, Tframe + Tbuffer, F1bin); double F1power = subMatrix[Tbuffer, F1bin]; // convert to vector var spectrum = MatrixTools.GetColumnAverages(subMatrix); // use the following code to get estimate of background noise double[,] powerMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - 3, 10, Tframe + 3, F1bin); double averagePower = (MatrixTools.GetRowAverages(powerMatrix)).Average(); double score = F1power - averagePower; // debug - checking what the spectrum looks like. //for (int i = 0; i < 18; i++) // spectrum[i] = -100.0; //DataTools.writeBarGraph(spectrum); // locate the peaks in lower frequency bands, F2 and F3 bool[] peaks = DataTools.GetPeaks(spectrum); int F2bin = 0; double F2power = -200.0; // dB for (int i = -3; i <= 2; i++) { int bin = F1bin - F1AndF2Gap + i; if ((peaks[bin]) && (F2power < subMatrix[1, bin])) { F2bin = bin; F2power = subMatrix[1, bin]; } } if (F2bin == 0) { continue; } if (F2power == -200.0) { continue; } score += (F2power - averagePower); int F3bin = 0; double F3power = -200.0; for (int i = -5; i <= 2; i++) { int bin = F1bin - F1AndF3Gap + i; if ((peaks[bin]) && (F3power < subMatrix[1, bin])) { F3bin = bin; F3power = subMatrix[1, bin]; } } if (F3bin == 0) { continue; } if (F3power == -200.0) { continue; } score += (F3power - averagePower); score /= 3; // ignore events where SNR < decibel threshold if (score < scoreThreshold) { continue; } // ignore events with wrong power distribution. A good LimnoConvex call has strongest F1 power if ((F3power > F1power) || (F2power > F1power)) { continue; } //freq Bin ID must be converted back to Matrix row ID // freqBin + rowID = binCount - 1; // therefore: rowID = binCount - freqBin - 1; minRowID = rhzRowCount - F1bin - 2; maxRowID = rhzRowCount - F3bin - 1; int F1RowID = rhzRowCount - F1bin - 1; int F2RowID = rhzRowCount - F2bin - 1; int F3RowID = rhzRowCount - F3bin - 1; int maxfreq = dominantFrequency + hzBuffer; int topBin = (int)Math.Round(maxfreq / herzPerBin); int frameCount = 4; double duration = frameCount * frameStepInSeconds; double startTimeWrtSegment = (Tframe - 2) * frameStepInSeconds; // Got to here so start initialising an acoustic event var ae = new AcousticEvent(segmentStartOffset, startTimeWrtSegment, duration, minimumFrequency, maxfreq); ae.SetTimeAndFreqScales(framesPerSec, herzPerBin); //var ae = new AcousticEvent(oblong, recording.Nyquist, binCount, frameDurationInSeconds, frameStepInSeconds, frameCount); //ae.StartOffset = TimeSpan.FromSeconds(Tframe * frameStepInSeconds); var pointF1 = new Point(2, topBin - F1bin); var pointF2 = new Point(2, topBin - F2bin); var pointF3 = new Point(2, topBin - F3bin); ae.Points = new List <Point>(); ae.Points.Add(pointF1); ae.Points.Add(pointF2); ae.Points.Add(pointF3); //tried using HitElements but did not do what I wanted later on. //ae.HitElements = new HashSet<Point>(); //ae.HitElements = new SortedSet<Point>(); //ae.HitElements.Add(pointF1); //ae.HitElements.Add(pointF2); //ae.HitElements.Add(pointF3); ae.Score = score; //ae.MinFreq = Math.Round((topBin - F3bin - 5) * herzPerBin); //ae.MaxFreq = Math.Round(topBin * herzPerBin); acousticEvents.Add(ae); } // now add in extra common info to the acoustic events acousticEvents.ForEach(ae => { ae.SpeciesName = configDict[AnalysisKeys.SpeciesName]; ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; ae.SegmentDurationSeconds = recording.Duration.TotalSeconds; ae.Name = abbreviatedName; ae.BorderColour = Color.Red; ae.FileName = recording.BaseName; }); double[] scores = new double[rhzColCount]; // predefinition of score array double nomalisationConstant = scoreThreshold * 4; // four times the score threshold double compressionFactor = rhzColCount / (double)sonogram.Data.GetLength(0); foreach (AcousticEvent ae in acousticEvents) { ae.ScoreNormalised = ae.Score / nomalisationConstant; if (ae.ScoreNormalised > 1.0) { ae.ScoreNormalised = 1.0; } int frameID = (int)Math.Round(ae.EventStartSeconds / frameDurationInSeconds); int hiresFrameID = (int)Math.Floor(frameID * compressionFactor); scores[hiresFrameID] = ae.ScoreNormalised; } var plot = new Plot(AnalysisName, scores, scoreThreshold); // DEBUG ONLY ################################ TEMPORARY ################################ // Draw a standard spectrogram and mark of hites etc. bool createStandardDebugSpectrogram = true; var imageDir = new DirectoryInfo(outputDir.FullName + @"\SpectrogramImages"); if (!imageDir.Exists) { imageDir.Create(); } if (createStandardDebugSpectrogram) { var fileName2 = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name); string filePath2 = Path.Combine(imageDir.FullName, fileName + ".Spectrogram.png"); Bitmap sonoBmp = (Bitmap)sonogram.GetImage(); int height = sonoBmp.Height; foreach (AcousticEvent ae in acousticEvents) { ae.DrawEvent(sonoBmp); //g.DrawRectangle(pen, ob.ColumnLeft, ob.RowTop, ob.ColWidth-1, ob.RowWidth); //ae.DrawPoint(sonoBmp, ae.HitElements.[0], Color.OrangeRed); //ae.DrawPoint(sonoBmp, ae.HitElements[1], Color.Yellow); //ae.DrawPoint(sonoBmp, ae.HitElements[2], Color.Green); ae.DrawPoint(sonoBmp, ae.Points[0], Color.OrangeRed); ae.DrawPoint(sonoBmp, ae.Points[1], Color.Yellow); ae.DrawPoint(sonoBmp, ae.Points[2], Color.LimeGreen); } // draw the original hits on the standard sonogram foreach (int[] array in newList) { sonoBmp.SetPixel(array[0], height - array[1], Color.Cyan); } // mark off every tenth frequency bin on the standard sonogram for (int r = 0; r < 20; r++) { sonoBmp.SetPixel(0, height - (r * 10) - 1, Color.Blue); sonoBmp.SetPixel(1, height - (r * 10) - 1, Color.Blue); } // mark off upper bound and lower frequency bound sonoBmp.SetPixel(0, height - dominantBinMin, Color.Lime); sonoBmp.SetPixel(0, height - dominantBinMax, Color.Lime); sonoBmp.Save(filePath2); } // END DEBUG ################################ TEMPORARY ################################ return(new LimnodynastesConvexResults { Sonogram = sonogram, Hits = null, Plot = plot, Events = acousticEvents, RecordingDuration = recording.Duration, }); } // Analysis()
/// <summary> /// The CORE ANALYSIS METHOD. /// </summary> public static Tuple <BaseSonogram, double[, ], Plot, List <AcousticEvent>, TimeSpan> Analysis(FileInfo fiSegmentOfSourceFile, Dictionary <string, string> configDict, TimeSpan segmentStartOffset) { //set default values - int frameLength = 1024; if (configDict.ContainsKey(AnalysisKeys.FrameLength)) { frameLength = int.Parse(configDict[AnalysisKeys.FrameLength]); } double windowOverlap = 0.0; int minHz = int.Parse(configDict["MIN_HZ"]); int minFormantgap = int.Parse(configDict["MIN_FORMANT_GAP"]); int maxFormantgap = int.Parse(configDict["MAX_FORMANT_GAP"]); double decibelThreshold = double.Parse(configDict["DECIBEL_THRESHOLD"]); //dB double harmonicIntensityThreshold = double.Parse(configDict["INTENSITY_THRESHOLD"]); //in 0-1 double callDuration = double.Parse(configDict["CALL_DURATION"]); // seconds AudioRecording recording = new AudioRecording(fiSegmentOfSourceFile.FullName); //i: MAKE SONOGRAM var sonoConfig = new SonogramConfig { SourceFName = recording.BaseName, WindowSize = frameLength, WindowOverlap = windowOverlap, NoiseReductionType = SNR.KeyToNoiseReductionType("STANDARD"), }; //default values config TimeSpan tsRecordingtDuration = recording.Duration; int sr = recording.SampleRate; double freqBinWidth = sr / (double)sonoConfig.WindowSize; double framesPerSecond = freqBinWidth; //the Xcorrelation-FFT technique requires number of bins to scan to be power of 2. //assuming sr=17640 and window=1024, then 64 bins span 1100 Hz above the min Hz level. i.e. 500 to 1600 //assuming sr=17640 and window=1024, then 128 bins span 2200 Hz above the min Hz level. i.e. 500 to 2700 int numberOfBins = 64; int minBin = (int)Math.Round(minHz / freqBinWidth) + 1; int maxbin = minBin + numberOfBins - 1; int maxHz = (int)Math.Round(minHz + (numberOfBins * freqBinWidth)); BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); int rowCount = sonogram.Data.GetLength(0); int colCount = sonogram.Data.GetLength(1); double[,] subMatrix = MatrixTools.Submatrix(sonogram.Data, 0, minBin, rowCount - 1, maxbin); int callSpan = (int)Math.Round(callDuration * framesPerSecond); //############################################################################################################################################# //ii: DETECT HARMONICS var results = CrossCorrelation.DetectHarmonicsInSonogramMatrix(subMatrix, decibelThreshold, callSpan); double[] dBArray = results.Item1; double[] intensity = results.Item2; //an array of periodicity scores double[] periodicity = results.Item3; //intensity = DataTools.filterMovingAverage(intensity, 3); int noiseBound = (int)(100 / freqBinWidth); //ignore 0-100 hz - too much noise double[] scoreArray = new double[intensity.Length]; for (int r = 0; r < rowCount; r++) { if (intensity[r] < harmonicIntensityThreshold) { continue; } //ignore locations with incorrect formant gap double herzPeriod = periodicity[r] * freqBinWidth; if (herzPeriod < minFormantgap || herzPeriod > maxFormantgap) { continue; } //find freq having max power and use info to adjust score. //expect humans to have max < 1000 Hz double[] spectrum = MatrixTools.GetRow(sonogram.Data, r); for (int j = 0; j < noiseBound; j++) { spectrum[j] = 0.0; } int maxIndex = DataTools.GetMaxIndex(spectrum); int freqWithMaxPower = (int)Math.Round(maxIndex * freqBinWidth); double discount = 1.0; if (freqWithMaxPower < 1200) { discount = 0.0; } if (intensity[r] > harmonicIntensityThreshold) { scoreArray[r] = intensity[r] * discount; } } //transfer info to a hits matrix. var hits = new double[rowCount, colCount]; double threshold = harmonicIntensityThreshold * 0.75; //reduced threshold for display of hits for (int r = 0; r < rowCount; r++) { if (scoreArray[r] < threshold) { continue; } double herzPeriod = periodicity[r] * freqBinWidth; for (int c = minBin; c < maxbin; c++) { //hits[r, c] = herzPeriod / (double)380; //divide by 380 to get a relativePeriod; hits[r, c] = (herzPeriod - minFormantgap) / maxFormantgap; //to get a relativePeriod; } } //iii: CONVERT TO ACOUSTIC EVENTS double maxPossibleScore = 0.5; int halfCallSpan = callSpan / 2; var predictedEvents = new List <AcousticEvent>(); for (int i = 0; i < rowCount; i++) { //assume one score position per crow call if (scoreArray[i] < 0.001) { continue; } double startTime = (i - halfCallSpan) / framesPerSecond; AcousticEvent ev = new AcousticEvent(segmentStartOffset, startTime, callDuration, minHz, maxHz); ev.SetTimeAndFreqScales(framesPerSecond, freqBinWidth); ev.Score = scoreArray[i]; ev.ScoreNormalised = ev.Score / maxPossibleScore; // normalised to the user supplied threshold //ev.Score_MaxPossible = maxPossibleScore; predictedEvents.Add(ev); } //for loop Plot plot = new Plot("CROW", intensity, harmonicIntensityThreshold); return(Tuple.Create(sonogram, hits, plot, predictedEvents, tsRecordingtDuration)); } //Analysis()
/// <summary> /// Do your analysis. This method is called once per segment (typically one-minute segments). /// </summary> /// <param name="recording"></param> /// <param name="configuration"></param> /// <param name="segmentStartOffset"></param> /// <param name="getSpectralIndexes"></param> /// <param name="outputDirectory"></param> /// <param name="imageWidth"></param> /// <returns></returns> public override RecognizerResults Recognize(AudioRecording recording, Config configuration, TimeSpan segmentStartOffset, Lazy <IndexCalculateResult[]> getSpectralIndexes, DirectoryInfo outputDirectory, int?imageWidth) { var recognizerConfig = new LitoriaNasutaConfig(); recognizerConfig.ReadConfigFile(configuration); // BETTER TO SET THESE. IGNORE USER! // this default framesize seems to work const int frameSize = 1024; const double windowOverlap = 0.0; // i: MAKE SONOGRAM var sonoConfig = new SonogramConfig { SourceFName = recording.BaseName, WindowSize = frameSize, WindowOverlap = windowOverlap, // use the default HAMMING window //WindowFunction = WindowFunctions.HANNING.ToString(), //WindowFunction = WindowFunctions.NONE.ToString(), // if do not use noise reduction can get a more sensitive recogniser. //NoiseReductionType = NoiseReductionType.None NoiseReductionType = NoiseReductionType.Standard, NoiseReductionParameter = 0.0, }; TimeSpan recordingDuration = recording.WavReader.Time; int sr = recording.SampleRate; double freqBinWidth = sr / (double)sonoConfig.WindowSize; int minBin = (int)Math.Round(recognizerConfig.MinHz / freqBinWidth) + 1; int maxBin = (int)Math.Round(recognizerConfig.MaxHz / freqBinWidth) + 1; var decibelThreshold = 3.0; BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // ###################################################################### // ii: DO THE ANALYSIS AND RECOVER SCORES OR WHATEVER int rowCount = sonogram.Data.GetLength(0); double[] amplitudeArray = MatrixTools.GetRowAveragesOfSubmatrix(sonogram.Data, 0, minBin, rowCount - 1, maxBin); //double[] topBand = MatrixTools.GetRowAveragesOfSubmatrix(sonogram.Data, 0, maxBin + 3, (rowCount - 1), maxBin + 9); //double[] botBand = MatrixTools.GetRowAveragesOfSubmatrix(sonogram.Data, 0, minBin - 3, (rowCount - 1), minBin - 9); // ii: DO THE ANALYSIS AND RECOVER SCORES OR WHATEVER var acousticEvents = AcousticEvent.ConvertScoreArray2Events( amplitudeArray, recognizerConfig.MinHz, recognizerConfig.MaxHz, sonogram.FramesPerSecond, freqBinWidth, decibelThreshold, recognizerConfig.MinDuration, recognizerConfig.MaxDuration, segmentStartOffset); double[,] hits = null; var prunedEvents = new List <AcousticEvent>(); acousticEvents.ForEach(ae => { ae.SpeciesName = recognizerConfig.SpeciesName; ae.SegmentDurationSeconds = recordingDuration.TotalSeconds; ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; ae.Name = recognizerConfig.AbbreviatedSpeciesName; }); var thresholdedPlot = new double[amplitudeArray.Length]; for (int x = 0; x < amplitudeArray.Length; x++) { if (amplitudeArray[x] > decibelThreshold) { thresholdedPlot[x] = amplitudeArray[x]; } } var maxDb = amplitudeArray.MaxOrDefault(); double[] normalisedScores; double normalisedThreshold; DataTools.Normalise(thresholdedPlot, decibelThreshold, out normalisedScores, out normalisedThreshold); var text = string.Format($"{this.DisplayName} (Fullscale={maxDb:f1}dB)"); var plot = new Plot(text, normalisedScores, normalisedThreshold); if (true) { // display a variety of debug score arrays DataTools.Normalise(amplitudeArray, decibelThreshold, out normalisedScores, out normalisedThreshold); var amplPlot = new Plot("Band amplitude", normalisedScores, normalisedThreshold); var debugPlots = new List <Plot> { plot, amplPlot }; // NOTE: This DrawDebugImage() method can be over-written in this class. var debugImage = DrawDebugImage(sonogram, acousticEvents, debugPlots, hits); var debugPath = FilenameHelpers.AnalysisResultPath(outputDirectory, recording.BaseName, this.SpeciesName, "png", "DebugSpectrogram"); debugImage.Save(debugPath); } return(new RecognizerResults() { Sonogram = sonogram, Hits = hits, Plots = plot.AsList(), Events = acousticEvents, }); }
internal RecognizerResults Gruntwork(AudioRecording audioRecording, Config configuration, DirectoryInfo outputDirectory, TimeSpan segmentStartOffset) { double noiseReductionParameter = configuration.GetDoubleOrNull(AnalysisKeys.NoiseBgThreshold) ?? 0.1; // make a spectrogram var config = new SonogramConfig { WindowSize = 256, NoiseReductionType = NoiseReductionType.Standard, NoiseReductionParameter = noiseReductionParameter, }; config.WindowOverlap = 0.0; // now construct the standard decibel spectrogram WITH noise removal, and look for LimConvex // get frame parameters for the analysis var sonogram = (BaseSonogram) new SpectrogramStandard(config, audioRecording.WavReader); // remove the DC column var spg = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.Data.GetLength(0) - 1, sonogram.Data.GetLength(1) - 1); int sampleRate = audioRecording.SampleRate; int rowCount = spg.GetLength(0); int colCount = spg.GetLength(1); int frameSize = config.WindowSize; int frameStep = frameSize; // this default = zero overlap double frameStepInSeconds = frameStep / (double)sampleRate; double framesPerSec = 1 / frameStepInSeconds; // reading in variables from the config file string speciesName = configuration[AnalysisKeys.SpeciesName] ?? "<no species>"; string abbreviatedSpeciesName = configuration[AnalysisKeys.AbbreviatedSpeciesName] ?? "<no.sp>"; int minHz = configuration.GetInt(AnalysisKeys.MinHz); int maxHz = configuration.GetInt(AnalysisKeys.MaxHz); // ## THREE THRESHOLDS ---- only one of these is given to user. // minimum dB to register a dominant freq peak. After noise removal double peakThresholdDb = 3.0; // The threshold dB amplitude in the dominant freq bin required to yield an event double eventThresholdDb = 6; // minimum score for an acceptable event - that is when processing the score array. double similarityThreshold = configuration.GetDoubleOrNull(AnalysisKeys.EventThreshold) ?? 0.2; // IMPORTANT: The following frame durations assume a sampling rate = 22050 and window size of 256. int minFrameWidth = 7; int maxFrameWidth = 14; double minDuration = (minFrameWidth - 1) * frameStepInSeconds; double maxDuration = maxFrameWidth * frameStepInSeconds; // Calculate Max Amplitude int binMin = (int)Math.Round(minHz / sonogram.FBinWidth); int binMax = (int)Math.Round(maxHz / sonogram.FBinWidth); int[] dominantBins = new int[rowCount]; // predefinition of events max frequency double[] scores = new double[rowCount]; // predefinition of score array double[,] hits = new double[rowCount, colCount]; // loop through all spectra/rows of the spectrogram - NB: spg is rotated to vertical. // mark the hits in hitMatrix for (int s = 0; s < rowCount; s++) { double[] spectrum = MatrixTools.GetRow(spg, s); double maxAmplitude = double.MinValue; int maxId = 0; // loop through bandwidth of L.onvex call and look for dominant frequency for (int binID = 5; binID < binMax; binID++) { if (spectrum[binID] > maxAmplitude) { maxAmplitude = spectrum[binID]; maxId = binID; } } if (maxId < binMin) { continue; } // peak should exceed thresold amplitude if (spectrum[maxId] < peakThresholdDb) { continue; } scores[s] = maxAmplitude; dominantBins[s] = maxId; // Console.WriteLine("Col {0}, Bin {1} ", c, freqBinID); } // loop through all spectra // Find average amplitude double[] amplitudeArray = MatrixTools.GetRowAveragesOfSubmatrix( sonogram.Data, 0, binMin, rowCount - 1, binMax); var highPassFilteredSignal = DspFilters.SubtractBaseline(amplitudeArray, 7); // We now have a list of potential hits for C. tinnula. This needs to be filtered. var startEnds = new List <Point>(); Plot.FindStartsAndEndsOfScoreEvents(highPassFilteredSignal, eventThresholdDb, minFrameWidth, maxFrameWidth, out var prunedScores, out startEnds); // High pass Filter // loop through the score array and find beginning and end of potential events var potentialEvents = new List <AcousticEvent>(); foreach (Point point in startEnds) { // get average of the dominant bin int binSum = 0; int binCount = 0; int eventWidth = point.Y - point.X + 1; for (int s = point.X; s <= point.Y; s++) { if (dominantBins[s] >= binMin) { binSum += dominantBins[s]; binCount++; } } // find average dominant bin for the event int avDominantBin = (int)Math.Round(binSum / (double)binCount); int avDominantFreq = (int)(Math.Round(binSum / (double)binCount) * sonogram.FBinWidth); // Get score for the event. // Use a simple template for the honk and calculate cosine similarity to the template. // Template has three dominant frequenices. // minimum number of bins covering frequency bandwidth of C. tinnula call// minimum number of bins covering frequency bandwidth of L.convex call int callBinWidth = 14; var templates = GetCtinnulaTemplates(callBinWidth); var eventMatrix = MatrixTools.Submatrix(spg, point.X, avDominantBin - callBinWidth + 2, point.Y, avDominantBin + 1); double eventScore = GetEventScore(eventMatrix, templates); // put hits into hits matrix // put cosine score into the score array for (int s = point.X; s <= point.Y; s++) { hits[s, avDominantBin] = 10; prunedScores[s] = eventScore; } if (eventScore < similarityThreshold) { continue; } int topBinForEvent = avDominantBin + 2; int bottomBinForEvent = topBinForEvent - callBinWidth; double startTime = point.X * frameStepInSeconds; double durationTime = eventWidth * frameStepInSeconds; var newEvent = new AcousticEvent(segmentStartOffset, startTime, durationTime, minHz, maxHz); newEvent.DominantFreq = avDominantFreq; newEvent.Score = eventScore; newEvent.SetTimeAndFreqScales(framesPerSec, sonogram.FBinWidth); newEvent.Name = string.Empty; // remove name because it hides spectral content of the event. potentialEvents.Add(newEvent); } // display the original score array scores = DataTools.normalise(scores); var debugPlot = new Plot(this.DisplayName, scores, similarityThreshold); // DEBUG IMAGE this recognizer only. MUST set false for deployment. bool displayDebugImage = MainEntry.InDEBUG; if (displayDebugImage) { // display a variety of debug score arrays DataTools.Normalise(amplitudeArray, eventThresholdDb, out var normalisedScores, out var normalisedThreshold); var ampltdPlot = new Plot("Average amplitude", normalisedScores, normalisedThreshold); DataTools.Normalise(highPassFilteredSignal, eventThresholdDb, out normalisedScores, out normalisedThreshold); var demeanedPlot = new Plot("Hi Pass", normalisedScores, normalisedThreshold); /* * DataTools.Normalise(scores, eventThresholdDb, out normalisedScores, out normalisedThreshold); * var ampltdPlot = new Plot("amplitude", normalisedScores, normalisedThreshold); * * * DataTools.Normalise(lowPassFilteredSignal, decibelThreshold, out normalisedScores, out normalisedThreshold); * var lowPassPlot = new Plot("Low Pass", normalisedScores, normalisedThreshold); */ var debugPlots = new List <Plot> { ampltdPlot, demeanedPlot }; Image debugImage = DisplayDebugImage(sonogram, potentialEvents, debugPlots, null); var debugPath = outputDirectory.Combine(FilenameHelpers.AnalysisResultName(Path.GetFileNameWithoutExtension(audioRecording.BaseName), this.Identifier, "png", "DebugSpectrogram")); debugImage.Save(debugPath.FullName); } // display the cosine similarity scores var plot = new Plot(this.DisplayName, prunedScores, similarityThreshold); var plots = new List <Plot> { plot }; // add names into the returned events foreach (AcousticEvent ae in potentialEvents) { ae.Name = "speciesName"; // abbreviatedSpeciesName; } return(new RecognizerResults() { Events = potentialEvents, Hits = hits, Plots = plots, Sonogram = sonogram, }); }
// OTHER CONSTANTS //private const string ImageViewer = @"C:\Windows\system32\mspaint.exe"; /// <summary> /// Do your analysis. This method is called once per segment (typically one-minute segments). /// </summary> /// <param name="recording"></param> /// <param name="configuration"></param> /// <param name="segmentStartOffset"></param> /// <param name="getSpectralIndexes"></param> /// <param name="outputDirectory"></param> /// <param name="imageWidth"></param> /// <returns></returns> public override RecognizerResults Recognize(AudioRecording recording, Config configuration, TimeSpan segmentStartOffset, Lazy <IndexCalculateResult[]> getSpectralIndexes, DirectoryInfo outputDirectory, int?imageWidth) { var recognizerConfig = new LitoriaWatjulumConfig(); recognizerConfig.ReadConfigFile(configuration); //int maxOscilRate = (int)Math.Ceiling(1 / lwConfig.MinPeriod); if (recording.WavReader.SampleRate != 22050) { throw new InvalidOperationException("Requires a 22050Hz file"); } TimeSpan recordingDuration = recording.WavReader.Time; // this default framesize seems to work const int frameSize = 128; double windowOverlap = 0.0; // calculate the overlap instead //double windowOverlap = Oscillations2012.CalculateRequiredFrameOverlap( // recording.SampleRate, // frameSize, // maxOscilRate); // i: MAKE SONOGRAM var sonoConfig = new SonogramConfig { SourceFName = recording.BaseName, //set default values - ignore those set by user WindowSize = frameSize, WindowOverlap = windowOverlap, // the default window is HAMMING //WindowFunction = WindowFunctions.HANNING.ToString(), //WindowFunction = WindowFunctions.NONE.ToString(), // if do not use noise reduction can get a more sensitive recogniser. //NoiseReductionType = NoiseReductionType.NONE, NoiseReductionType = SNR.KeyToNoiseReductionType("STANDARD"), }; //############################################################################################################################################# //DO THE ANALYSIS var results = Analysis(recording, sonoConfig, recognizerConfig, MainEntry.InDEBUG, segmentStartOffset); //###################################################################### if (results == null) { return(null); //nothing to process } var sonogram = results.Item1; var hits = results.Item2; var scoreArray = results.Item3; var predictedEvents = results.Item4; var debugImage = results.Item5; // old way of creating a path: //var debugPath = outputDirectory.Combine(FilenameHelpers.AnalysisResultName(Path.GetFileNameWithoutExtension(recording.FileName), SpeciesName, "png", "DebugSpectrogram")); var debugPath = FilenameHelpers.AnalysisResultPath(outputDirectory, recording.BaseName, this.SpeciesName, "png", "DebugSpectrogram"); debugImage.Save(debugPath); //############################################################################################################################################# // Prune events here if required i.e. remove those below threshold score if this not already done. See other recognizers. foreach (var ae in predictedEvents) { // add additional info ae.Name = recognizerConfig.AbbreviatedSpeciesName; ae.SpeciesName = recognizerConfig.SpeciesName; ae.SegmentDurationSeconds = recordingDuration.TotalSeconds; ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; } // do a recognizer TEST. if (false) { var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName); TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray); AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents); } var plot = new Plot(this.DisplayName, scoreArray, recognizerConfig.EventThreshold); return(new RecognizerResults() { Sonogram = sonogram, Hits = hits, Plots = plot.AsList(), Events = predictedEvents, }); }