} //GetExcludeRules()
//filter events using the exclude rules - just changes the event's normalised score
public static AcousticEvent FilterEvent(AcousticEvent ae, List <string[]> rules)
{
//loop through exclusion rules - DO NOT DELETE events - set score to zero so can check later what is happening.
foreach (string[] rule in rules)
{
string feature = rule[0];
string op = rule[1];
double value = double.Parse(rule[2]);
//if (feature == key_BANDWIDTH_SCORE)
//{
// if ((op == "LT" && ae.kiwi_bandWidthScore < value) || (op == "GT" && ae.kiwi_bandWidthScore > value))
// {
// ae.kiwi_bandWidthScore = 0.0;
// ae.ScoreNormalised = 0.0;
// return ae;
// }
//}
//else // end if key_BANDWIDTH_SCORE
// if (feature == key_INTENSITY_SCORE)
// {
// if ((op == "LT" && ae.kiwi_intensityScore < value) || (op == "GT" && ae.kiwi_intensityScore > value))
// {
// ae.ScoreNormalised = 0.0;
// return ae;
// }
// } // end if key_INTENSITY_SCORE
}
return(ae);
}
/// <summary>
///
/// </summary>
/// <param name="ae">an acoustic event</param>
/// <param name="dbArray">The sequence of frame dB over the event</param>
/// <returns></returns>
public static System.Tuple <double, double> KiwiPeakAnalysis(AcousticEvent ae, double[] dbArray)
{
//dbArray = DataTools.filterMovingAverage(dbArray, 3);
bool[] peaks = DataTools.GetPeaks(dbArray); //locate the peaks
double[] peakValues = new double[dbArray.Length];
for (int i = 0; i < dbArray.Length; i++)
{
if (peaks[i])
{
peakValues[i] = dbArray[i];
}
}
//take the top N peaks
int N = 5;
double[] topNValues = new double[N];
for (int p = 0; p < N; p++)
{
int maxID = DataTools.GetMaxIndex(peakValues);
topNValues[p] = peakValues[maxID];
peakValues[maxID] = 0.0;
}
//PROCESS PEAK DECIBELS
double avPeakDB, sdPeakDB;
NormalDist.AverageAndSD(topNValues, out avPeakDB, out sdPeakDB);
return(System.Tuple.Create(avPeakDB, sdPeakDB));
}
public static List <AcousticEvent> ReadGroundParrotTemplateAsList(BaseSonogram sonogram)
{
var timeScale = sonogram.FrameStep;
var hzScale = (int)sonogram.FBinWidth;
int rows = GroundParrotTemplate1.GetLength(0);
double timeOffset = GroundParrotTemplate1[0, 0];
var gpTemplate = new List <AcousticEvent>();
for (int r = 0; r < rows; r++)
{
int t1 = (int)Math.Round((GroundParrotTemplate1[r, 0] - timeOffset) / timeScale);
int t2 = (int)Math.Round((GroundParrotTemplate1[r, 1] - timeOffset) / timeScale);
int f2 = (int)Math.Round(GroundParrotTemplate1[r, 2] / hzScale);
int f1 = (int)Math.Round(GroundParrotTemplate1[r, 3] / hzScale);
Oblong o = new Oblong(t1, f1, t2, f2);
var ae = AcousticEvent.InitializeAcousticEvent(
TimeSpan.Zero,
o,
sonogram.NyquistFrequency,
sonogram.Configuration.FreqBinCount,
sonogram.FrameDuration,
sonogram.FrameStep,
sonogram.FrameCount);
gpTemplate.Add(ae);
}
return(gpTemplate);
}
public static double[,] ExtractFreqSubband(double[,] m, int minHz, int maxHz, bool doMelscale, int binCount, double binWidth)
{
int c1;
int c2;
AcousticEvent.Freq2BinIDs(doMelscale, minHz, maxHz, binCount, binWidth, out c1, out c2);
return(DataTools.Submatrix(m, 0, c1, m.GetLength(0) - 1, c2));
}
static CsvTests()
{
// pump static class initializers - it seems when running multiple tests that sometimes
// these classes are not discovered.
_ = new AcousticEvent();
_ = new ImportedEvent();
}
/// <summary>
/// FINDS OSCILLATIONS IN A SONOGRAM
/// But first it segments the sonogram based on acoustic energy in freq band of interest.
/// </summary>
/// <param name="sonogram">sonogram derived from the recording.</param>
/// <param name="minHz">min bound freq band to search.</param>
/// <param name="maxHz">max bound freq band to search.</param>
/// <param name="dctDuration">duration of DCT in seconds.</param>
/// <param name="dctThreshold">minimum amplitude of DCT. </param>
/// <param name="minOscilFreq">ignore oscillation frequencies below this threshold.</param>
/// <param name="maxOscilFreq">ignore oscillation frequencies greater than this. </param>
/// <param name="scoreThreshold">used for FP/FN.</param>
/// <param name="minDuration">ignore hits whose duration is shorter than this.</param>
/// <param name="maxDuration">ignore hits whose duration is longer than this.</param>
/// <param name="scores">return an array of scores over the entire recording.</param>
/// <param name="events">return a list of acoustic events.</param>
/// <param name="hits">a matrix that show where there is an oscillation of sufficient amplitude in the correct range.
/// Values in the matrix are the oscillation rate. i.e. if OR = 2.0 = 2 oscillations per second. </param>
public static void Execute(SpectrogramStandard sonogram, bool doSegmentation, int minHz, int maxHz,
double dctDuration, double dctThreshold, bool normaliseDCT, int minOscilFreq, int maxOscilFreq,
double scoreThreshold, double minDuration, double maxDuration,
out double[] scores, out List <AcousticEvent> events, out double[,] hits, out double[] intensity,
out TimeSpan totalTime)
{
DateTime startTime1 = DateTime.Now;
//EXTRACT SEGMENTATIOn EVENTS
//DO SEGMENTATION
double smoothWindow = 1 / (double)minOscilFreq; //window = max oscillation period
Log.WriteLine(" Segmentation smoothing window = {0:f2} seconds", smoothWindow);
double thresholdSD = 0.1; //Set threshold to 1/5th of a standard deviation of the background noise.
maxDuration = double.MaxValue; //Do not constrain maximum length of events.
var tuple = AcousticEvent.GetSegmentationEvents(sonogram, doSegmentation, TimeSpan.Zero, minHz, maxHz, smoothWindow, thresholdSD, minDuration, maxDuration);
var segmentEvents = tuple.Item1;
intensity = tuple.Item5;
Log.WriteLine("Number of segments={0}", segmentEvents.Count);
TimeSpan span1 = DateTime.Now.Subtract(startTime1);
Log.WriteLine(" SEGMENTATION COMP TIME = " + span1.TotalMilliseconds.ToString() + "ms");
DateTime startTime2 = DateTime.Now;
//DETECT OSCILLATIONS
hits = DetectOscillationsInSonogram(sonogram, minHz, maxHz, dctDuration, dctThreshold, normaliseDCT, minOscilFreq, maxOscilFreq, segmentEvents);
hits = RemoveIsolatedOscillations(hits);
//EXTRACT SCORES AND ACOUSTIC EVENTS
scores = GetOscillationScores(hits, minHz, maxHz, sonogram.FBinWidth);
double[] oscFreq = GetOscillationFrequency(hits, minHz, maxHz, sonogram.FBinWidth);
events = ConvertODScores2Events(
scores,
oscFreq,
minHz,
maxHz,
sonogram.FramesPerSecond,
sonogram.FBinWidth,
sonogram.Configuration.FreqBinCount,
scoreThreshold,
minDuration,
maxDuration,
sonogram.Configuration.SourceFName,
TimeSpan.Zero);
//events = segmentEvents; //#################################### to see segment events in output image.
DateTime endTime2 = DateTime.Now;
TimeSpan span2 = endTime2.Subtract(startTime2);
Log.WriteLine(" TOTAL COMP TIME = " + span2.ToString() + "s");
totalTime = endTime2.Subtract(startTime1);
}
public void TestEventMerging()
{
// make a list of events
var events = new List <AcousticEvent>();
double maxPossibleScore = 10.0;
var event1 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 1.0, eventDuration: 5.0, minFreq: 1000, maxFreq: 8000)
{
Name = "Event1",
Score = 1.0,
ScoreNormalised = 1.0 / maxPossibleScore,
};
events.Add(event1);
var event2 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 4.5, eventDuration: 2.0, minFreq: 1500, maxFreq: 6000)
{
Name = "Event2",
Score = 5.0,
ScoreNormalised = 5.0 / maxPossibleScore,
};
events.Add(event2);
var event3 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 7.0, eventDuration: 2.0, minFreq: 1000, maxFreq: 8000)
{
Name = "Event3",
Score = 9.0,
ScoreNormalised = 9.0 / maxPossibleScore,
};
events.Add(event3);
// combine adjacent acoustic events
events = AcousticEvent.CombineOverlappingEvents(events: events, segmentStartOffset: TimeSpan.Zero);
Assert.AreEqual(2, events.Count);
Assert.AreEqual(1.0, events[0].EventStartSeconds, 1E-4);
Assert.AreEqual(6.5, events[0].EventEndSeconds, 1E-4);
Assert.AreEqual(7.0, events[1].EventStartSeconds, 1E-4);
Assert.AreEqual(9.0, events[1].EventEndSeconds, 1E-4);
Assert.AreEqual(1000, events[0].LowFrequencyHertz);
Assert.AreEqual(8000, events[0].HighFrequencyHertz);
Assert.AreEqual(1000, events[1].LowFrequencyHertz);
Assert.AreEqual(8000, events[1].HighFrequencyHertz);
Assert.AreEqual(5.0, events[0].Score, 1E-4);
Assert.AreEqual(9.0, events[1].Score, 1E-4);
Assert.AreEqual(0.5, events[0].ScoreNormalised, 1E-4);
Assert.AreEqual(0.9, events[1].ScoreNormalised, 1E-4);
}
public static Tuple <BaseSonogram, AcousticEvent, double[, ], double[], double[, ]> Execute_Extraction(
AudioRecording recording,
double eventStart,
double eventEnd,
int minHz,
int maxHz,
double frameOverlap,
double backgroundThreshold,
TimeSpan segmentStartOffset)
{
//ii: MAKE SONOGRAM
SonogramConfig sonoConfig = new SonogramConfig(); //default values config
sonoConfig.SourceFName = recording.BaseName;
//sonoConfig.WindowSize = windowSize;
sonoConfig.WindowOverlap = frameOverlap;
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
Log.WriteLine("Frames: Size={0}, Count={1}, Duration={2:f1}ms, Overlap={5:f2}%, Offset={3:f1}ms, Frames/s={4:f1}",
sonogram.Configuration.WindowSize, sonogram.FrameCount, (sonogram.FrameDuration * 1000),
(sonogram.FrameStep * 1000), sonogram.FramesPerSecond, frameOverlap);
int binCount = (int)(maxHz / sonogram.FBinWidth) - (int)(minHz / sonogram.FBinWidth) + 1;
Log.WriteIfVerbose("Freq band: {0} Hz - {1} Hz. (Freq bin count = {2})", minHz, maxHz, binCount);
//calculate the modal noise profile
double SD_COUNT = 0.0; // number of noise standard deviations used to calculate noise threshold
NoiseProfile profile = NoiseProfile.CalculateModalNoiseProfile(sonogram.Data, SD_COUNT); //calculate modal noise profile
double[] modalNoise = DataTools.filterMovingAverage(profile.NoiseMode, 7); //smooth the noise profile
//extract modal noise values of the required event
double[] noiseSubband = SpectrogramTools.ExtractModalNoiseSubband(modalNoise, minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth);
//extract data values of the required event
double[,] target = SpectrogramTools.ExtractEvent(sonogram.Data, eventStart, eventEnd, sonogram.FrameStep,
minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth);
// create acoustic event with defined boundaries
AcousticEvent ae = new AcousticEvent(segmentStartOffset, eventStart, eventEnd - eventStart, minHz, maxHz);
ae.SetTimeAndFreqScales(sonogram.FramesPerSecond, sonogram.FBinWidth);
//truncate noise
sonogram.Data = SNR.TruncateBgNoiseFromSpectrogram(sonogram.Data, modalNoise);
sonogram.Data = SNR.RemoveNeighbourhoodBackgroundNoise(sonogram.Data, backgroundThreshold);
double[,] targetMinusNoise = SpectrogramTools.ExtractEvent(sonogram.Data, eventStart, eventEnd, sonogram.FrameStep,
minHz, maxHz, false, sonogram.NyquistFrequency, sonogram.FBinWidth);
return(Tuple.Create(sonogram, ae, target, noiseSubband, targetMinusNoise));
}
/// <summary>
/// Extracts an acoustic event from a sonogram given the location of a user defined rectangular marquee.
/// NOTE: Nyquist value is used ONLY if using mel scale.
/// </summary>
/// <param name="m">the sonogram data as matrix of reals</param>
/// <param name="start">start time in seconds</param>
/// <param name="end">end time in seconds</param>
/// <param name="frameOffset">the time scale: i.e. the duration in seconds of each frame</param>
/// <param name="minHz">lower freq bound of the event</param>
/// <param name="maxHz">upper freq bound of the event</param>
/// <param name="doMelscale">informs whether the sonogram data is linear or mel scale</param>
/// <param name="nyquist">full freq range 0-Nyquist</param>
/// <param name="binWidth">the frequency scale i.e. herz per bin width - assumes linear scale</param>
/// <returns></returns>
public static double[,] ExtractEvent(double[,] m, double start, double end, double frameOffset,
int minHz, int maxHz, bool doMelscale, int nyquist, double binWidth)
{
int r1;
int r2;
AcousticEvent.Time2RowIDs(start, end - start, frameOffset, out r1, out r2);
int c1;
int c2;
AcousticEvent.Freq2BinIDs(doMelscale, minHz, maxHz, nyquist, binWidth, out c1, out c2);
return(DataTools.Submatrix(m, r1, c1, r2, c2));
}
static CsvTests()
{
// pump static class initializers - it seems when running multiple tests that sometimes
// these classes are not discovered.
AcousticEvent aev = new AcousticEvent();
ImportedEvent iev = new ImportedEvent();
var configuration = Csv.DefaultConfiguration;
IDictionary csvMaps = (IDictionary)configuration.Maps.GetFieldValue("data");
Debug.WriteLine("initializing static types" + aev + iev + csvMaps.Count);
}
public static SpectralEvent ConvertAcousticEventToSpectralEvent(this AcousticEvent ae)
{
var segmentOffset = TimeSpan.FromSeconds(ae.SegmentStartSeconds);
var eventStart = ae.EventStartSeconds;
var eventEnd = ae.EventEndSeconds;
return(new SpectralEvent(segmentStartOffset: segmentOffset, eventStartRecordingRelative: eventStart, eventEndRecordingRelative: eventEnd, ae.LowFrequencyHertz, ae.HighFrequencyHertz)
{
Name = ae.Name,
//SegmentStartSeconds = ae.SegmentStartSeconds,
SegmentDurationSeconds = ae.SegmentDurationSeconds,
});
}
public static double[] ExtractModalNoiseSubband(double[] modalNoise, int minHz, int maxHz, bool doMelScale, int nyquist, double binWidth)
{
//extract subband modal noise profile
AcousticEvent.ConvertHertzToFrequencyBin(doMelScale, minHz, maxHz, nyquist, binWidth, out var c1, out var c2);
int subbandCount = c2 - c1 + 1;
var subband = new double[subbandCount];
for (int i = 0; i < subbandCount; i++)
{
subband[i] = modalNoise[c1 + i];
}
return(subband);
}
public static AcousticEvent ConvertSpectralEventToAcousticEvent(this SpectralEvent se)
{
var segmentStartOffset = TimeSpan.FromSeconds(se.SegmentStartSeconds);
double startTime = se.EventStartSeconds;
double duration = se.EventDurationSeconds;
double minHz = se.HighFrequencyHertz;
double maxHz = se.HighFrequencyHertz;
var ae = new AcousticEvent(segmentStartOffset, startTime, duration, minHz, maxHz)
{
Name = se.Name,
SegmentDurationSeconds = se.SegmentDurationSeconds,
};
return(ae);
}
/// <summary>
/// Detect using EPR.
/// </summary>
/// <param name="wavFilePath">
/// The wav file path.
/// </param>
/// <param name="eprNormalisedMinScore">
/// The epr Normalised Min Score.
/// </param>
/// <returns>
/// Tuple containing base Sonogram and list of acoustic events.
/// </returns>
public static Tuple <BaseSonogram, List <AcousticEvent> > Detect(FileInfo wavFilePath, Aed.AedConfiguration aedConfiguration, double eprNormalisedMinScore, TimeSpan segmentStartOffset)
{
Tuple <EventCommon[], AudioRecording, BaseSonogram> aed = Aed.Detect(wavFilePath, aedConfiguration, segmentStartOffset);
var events = aed.Item1;
var newEvents = new List <AcousticEvent>();
foreach (var be in events)
{
newEvents.Add(EventConverters.ConvertSpectralEventToAcousticEvent((SpectralEvent)be));
}
var aeEvents = newEvents.Select(ae => Util.fcornersToRect(ae.TimeStart, ae.TimeEnd, ae.HighFrequencyHertz, ae.LowFrequencyHertz)).ToList();
Log.Debug("EPR start");
var eprRects = EventPatternRecog.DetectGroundParrots(aeEvents, eprNormalisedMinScore);
Log.Debug("EPR finished");
var sonogram = aed.Item3;
SonogramConfig config = sonogram.Configuration;
double framesPerSec = 1 / config.GetFrameOffset(); // Surely this should go somewhere else
double freqBinWidth = config.NyquistFreq / (double)config.FreqBinCount;
// TODO this is common with AED
var eprEvents = new List <AcousticEvent>();
foreach (var rectScore in eprRects)
{
var ae = new AcousticEvent(
segmentStartOffset,
rectScore.Item1.Left,
rectScore.Item1.Right - rectScore.Item1.Left,
rectScore.Item1.Bottom,
rectScore.Item1.Top);
ae.SetTimeAndFreqScales(framesPerSec, freqBinWidth);
ae.SetTimeAndFreqScales(sonogram.NyquistFrequency, sonogram.Configuration.WindowSize, 0);
ae.SetScores(rectScore.Item2, 0, 1);
ae.BorderColour = aedConfiguration.AedEventColor;
ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds;
ae.SegmentDurationSeconds = aed.Item2.Duration.TotalSeconds;
eprEvents.Add(ae);
}
return(Tuple.Create(sonogram, eprEvents));
}
public void TestSonogramWithEventsOverlay()
{
// make a substitute sonogram image
var substituteSonogram = Drawing.NewImage(100, 256, Color.Black);
// make a list of events
var framesPerSecond = 10.0;
var freqBinWidth = 43.0664;
double maxPossibleScore = 10.0;
var events = new List <AcousticEvent>();
var event1 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 1.0, eventDuration: 5.0, minFreq: 1000, maxFreq: 8000)
{
Score = 10.0,
Name = "Event1",
ScoreNormalised = 10.0 / maxPossibleScore,
};
events.Add(event1);
var event2 = new AcousticEvent(segmentStartOffset: TimeSpan.Zero, eventStartSegmentRelative: 7.0, eventDuration: 2.0, minFreq: 1000, maxFreq: 8000)
{
Score = 1.0,
Name = "Event2",
ScoreNormalised = 1.0 / maxPossibleScore,
};
events.Add(event2);
// now add events into the spectrogram image.
// set color for the events
foreach (AcousticEvent ev in events)
{
// because we are testing placement of box not text.
ev.Name = string.Empty;
ev.BorderColour = AcousticEvent.DefaultBorderColor;
ev.ScoreColour = AcousticEvent.DefaultScoreColor;
ev.DrawEvent(substituteSonogram, framesPerSecond, freqBinWidth, 256);
}
this.ActualImage = substituteSonogram;
// BUG: this asset is faulty. See https://github.com/QutEcoacoustics/audio-analysis/issues/300#issuecomment-601537263
this.ExpectedImage = Image.Load <Rgb24>(
PathHelper.ResolveAssetPath("AcousticEventTests_SuperimposeEventsOnImage.png"));
this.AssertImagesEqual();
}
public static void DrawSonogram(BaseSonogram sonogram, string path, AcousticEvent ae)
{
Log.WriteLine("# Start to draw image of sonogram.");
bool doHighlightSubband = false; bool add1kHzLines = true;
using (Image img = sonogram.GetImage(doHighlightSubband, add1kHzLines))
using (Image_MultiTrack image = new Image_MultiTrack(img))
{
//img.Save(@"C:\SensorNetworks\WavFiles\temp1\testimage1.jpg", System.Drawing.Imaging.ImageFormat.Jpeg);
image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond));
image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram));
var aes = new List <AcousticEvent>();
aes.Add(ae);
image.AddEvents(aes, sonogram.NyquistFrequency, sonogram.Configuration.FreqBinCount, sonogram.FramesPerSecond);
image.Save(path);
}
} //end DrawSonogram
public void TestBaseTypesAreSerializedAsEnumerableAcousticEvent()
{
var exampleEvent = new AcousticEvent(100.Seconds(), 15, 4, 100, 3000);
var exampleEvent2 = new AcousticEvent(100.Seconds(), 15, 4, 100, 3000);
AcousticEvent[] childArray = { exampleEvent, exampleEvent2 };
EventBase[] baseArray = { exampleEvent, exampleEvent2 };
Csv.WriteToCsv(this.testFile, childArray);
var childText = File.ReadAllText(this.testFile.FullName);
Csv.WriteToCsv(this.testFile, baseArray);
var baseText = File.ReadAllText(this.testFile.FullName);
Assert.AreNotEqual(childText, baseText);
}
public void TestGetEventsAroundMaxima()
{
//string abbreviatedSpeciesName = "Pteropus";
string speciesName = "Pteropus species";
int minHz = 800;
int maxHz = 8000;
var minTimeSpan = TimeSpan.FromSeconds(0.15);
var maxTimeSpan = TimeSpan.FromSeconds(0.8);
double decibelThreshold = 9.0;
TimeSpan segmentStartOffset = TimeSpan.Zero;
var decibelArray = SNR.CalculateFreqBandAvIntensity(this.sonogram.Data, minHz, maxHz, this.sonogram.NyquistFrequency);
// prepare plots
double intensityNormalisationMax = 3 * decibelThreshold;
var eventThreshold = decibelThreshold / intensityNormalisationMax;
var normalisedIntensityArray = DataTools.NormaliseInZeroOne(decibelArray, 0, intensityNormalisationMax);
var plot = new Plot(speciesName + " Territory", normalisedIntensityArray, eventThreshold);
var plots = new List <Plot> {
plot
};
//iii: CONVERT decibel SCORES TO ACOUSTIC EVENTS
var acousticEvents = AcousticEvent.GetEventsAroundMaxima(
decibelArray,
segmentStartOffset,
minHz,
maxHz,
decibelThreshold,
minTimeSpan,
maxTimeSpan,
this.sonogram.FramesPerSecond,
this.sonogram.FBinWidth);
Assert.AreEqual(10, acousticEvents.Count);
Assert.AreEqual(new Rectangle(19, 1751, 168, 27), acousticEvents[0].GetEventAsRectangle());
Assert.AreEqual(new Rectangle(19, 1840, 168, 10), acousticEvents[2].GetEventAsRectangle());
Assert.AreEqual(new Rectangle(19, 1961, 168, 31), acousticEvents[5].GetEventAsRectangle());
Assert.AreEqual(new Rectangle(19, 2294, 168, 17), acousticEvents[7].GetEventAsRectangle());
Assert.AreEqual(new Rectangle(19, 2504, 168, 7), acousticEvents[9].GetEventAsRectangle());
//Assert.AreEqual(28.Seconds() + segmentOffset, stats.ResultStartSeconds.Seconds());
}
public static double CalculateAverageEventScore(AcousticEvent ae, double[] scoreArray)
{
int start = ae.Oblong.RowTop;
int end = ae.Oblong.RowBottom;
if (end > scoreArray.Length)
{
end = scoreArray.Length - 1;
}
int length = end - start + 1;
double sum = 0.0;
for (int i = start; i <= end; i++)
{
sum += scoreArray[i];
}
return(sum / length);
}
} // FELTWithSprTemplate()
/// <summary>
/// Edits the passed events to adjust start and end locations to position of maximum score
/// Also correct the scores because we want the max match for a template
/// Also correct the time scale
/// </summary>
/// <param name="matchEvents"></param>
/// <param name="callName"></param>
/// <param name="templateDuration"></param>
/// <param name="sonogramDuration"></param>
public static void AdjustEventLocation(List <AcousticEvent> matchEvents, string callName, double templateDuration, double sonogramDuration)
{
Log.WriteLine("# ADJUST EVENT LOCATIONS");
Log.WriteLine("# Event: " + callName);
foreach (AcousticEvent ae in matchEvents)
{
Log.WriteLine("# Old event frame= {0} to {1}.", ae.Oblong.RowTop, ae.Oblong.RowBottom);
ae.Name = callName;
ae.TimeStart = ae.Score_TimeOfMaxInEvent;
ae.TimeEnd = ae.TimeStart + templateDuration;
if (ae.TimeEnd > sonogramDuration)
{
ae.TimeEnd = sonogramDuration; // check for overflow.
}
ae.Oblong = AcousticEvent.ConvertEvent2Oblong(ae);
ae.Score = ae.Score_MaxInEvent;
ae.ScoreNormalised = ae.Score / ae.Score_MaxPossible; // normalised to the user supplied threshold
Log.WriteLine("# New event time = {0:f2} to {1:f2}.", ae.TimeStart, ae.TimeEnd);
Log.WriteLine("# New event frame= {0} to {1}.", ae.Oblong.RowTop, ae.Oblong.RowBottom);
}
}
/// <summary>
///
/// </summary>
/// <param name="wavPath"></param>
/// <param name="minHz"></param>
/// <param name="maxHz"></param>
/// <param name="frameOverlap"></param>
/// <param name="threshold"></param>
/// <param name="minDuration">used for smoothing intensity as well as for removing short events</param>
/// <param name="maxDuration"></param>
/// <returns></returns>
public static Tuple <BaseSonogram, List <AcousticEvent>, double, double, double, double[]> Execute_Segmentation(FileInfo wavPath,
int minHz, int maxHz, double frameOverlap, double smoothWindow, double thresholdSD, double minDuration, double maxDuration)
{
//i: GET RECORDING
AudioRecording recording = new AudioRecording(wavPath.FullName);
//ii: MAKE SONOGRAM
Log.WriteLine("# Start sonogram.");
SonogramConfig sonoConfig = new SonogramConfig(); //default values config
sonoConfig.WindowOverlap = frameOverlap;
sonoConfig.SourceFName = recording.BaseName;
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
//iii: DETECT SEGMENTS
Log.WriteLine("# Start event detection");
var tuple = AcousticEvent.GetSegmentationEvents((SpectrogramStandard)sonogram, TimeSpan.Zero, minHz, maxHz, smoothWindow,
thresholdSD, minDuration, maxDuration);
var tuple2 = Tuple.Create(sonogram, tuple.Item1, tuple.Item2, tuple.Item3, tuple.Item4, tuple.Item5);
return(tuple2);
}//end Execute_Segmentation
public static List <AcousticEvent> ConvertTracks2Events(
List <SpectralTrack> tracks,
TimeSpan segmentStartOffset)
/*, double framesPerSecond, double herzPerBin*/
{
if (tracks == null)
{
return(null);
}
var list = new List <AcousticEvent>();
if (tracks.Count == 0)
{
return(list);
}
foreach (SpectralTrack track in tracks)
{
double startTime = track.StartFrame / track.framesPerSecond;
int frameDuration = track.EndFrame - track.StartFrame + 1;
double duration = frameDuration / track.framesPerSecond;
double minFreq = track.herzPerBin * (track.AverageBin - 1);
double maxFreq = track.herzPerBin * (track.AverageBin + 1);
AcousticEvent ae = new AcousticEvent(segmentStartOffset, startTime, duration, minFreq, maxFreq);
ae.SetTimeAndFreqScales(track.framesPerSecond, track.herzPerBin);
ae.Name = "";
ae.BorderColour = Color.Blue;
ae.DominantFreq = track.AverageBin * track.herzPerBin;
ae.Periodicity = track.avPeriodicity;
ae.Score = track.avPeriodicityScore;
list.Add(ae);
}
return(list);
}
public static List <AcousticEvent> PruneOverlappingEvents(List <AcousticEvent> events, int percentOverlap)
{
double thresholdOverlap = percentOverlap / 100.0;
int count = events.Count;
for (int i = 0; i < count - 1; i++)
{
for (int j = i + 1; j < count; j++)
{
if (AcousticEvent.EventFractionalOverlap(events[i], events[j]) > thresholdOverlap)
{
//determine the event with highest score
if (events[i].Score >= events[j].Score)
{
events[j].Name = null;
}
else
{
events[i].Name = null;
}
}
}
}
List <AcousticEvent> pruned = new List <AcousticEvent>();
foreach (AcousticEvent ae in events)
{
if (ae.Name != null)
{
pruned.Add(ae);
}
}
return(pruned);
}
public void TestAcousticEventClassMap()
{
var ae = new AcousticEvent();
var result = new StringBuilder();
using (var str = new StringWriter(result))
{
var writer = new CsvWriter(str, Csv.DefaultConfiguration);
writer.WriteRecords(records: new[] { ae });
}
var actual = result.ToString();
foreach (var property in AcousticEvent.AcousticEventClassMap.IgnoredProperties.Except(AcousticEvent.AcousticEventClassMap.RemappedProperties))
{
Assert.IsFalse(
actual.Contains(property, StringComparison.InvariantCultureIgnoreCase),
$"output CSV should not contain text '{property}'.{Environment.NewLine}Actual: {actual}");
}
StringAssert.Contains(actual, "EventEndSeconds");
}
public static double[,] ExtractFreqSubband(double[,] m, int minHz, int maxHz, bool doMelscale, int binCount, double binWidth)
{
AcousticEvent.ConvertHertzToFrequencyBin(doMelscale, minHz, maxHz, binCount, binWidth, out var c1, out var c2);
return(DataTools.Submatrix(m, 0, c1, m.GetLength(0) - 1, c2));
}
/// <summary>
/// THE KEY ANALYSIS METHOD
/// </summary>
/// <param name="recording">
/// The segment Of Source File.
/// </param>
/// <param name="configDict">
/// The config Dict.
/// </param>
/// <param name="value"></param>
/// <returns>
/// The <see cref="LimnodynastesConvexResults"/>.
/// </returns>
internal static LimnodynastesConvexResults Analysis(
Dictionary <string, double[, ]> dictionaryOfHiResSpectralIndices,
AudioRecording recording,
Dictionary <string, string> configDict,
AnalysisSettings analysisSettings,
SegmentSettingsBase segmentSettings)
{
// for Limnodynastes convex, in the D.Stewart CD, there are peaks close to:
//1. 1950 Hz
//2. 1460 hz
//3. 970 hz These are 490 Hz apart.
// for Limnodynastes convex, in the JCU recording, there are peaks close to:
//1. 1780 Hz
//2. 1330 hz
//3. 880 hz These are 450 Hz apart.
// So strategy is to look for three peaks separated by same amount and in the vicinity of the above,
// starting with highest power (the top peak) and working down to lowest power (bottom peak).
var outputDir = segmentSettings.SegmentOutputDirectory;
TimeSpan segmentStartOffset = segmentSettings.SegmentStartOffset;
//KeyValuePair<string, double[,]> kvp = dictionaryOfHiResSpectralIndices.First();
var spg = dictionaryOfHiResSpectralIndices["RHZ"];
int rhzRowCount = spg.GetLength(0);
int rhzColCount = spg.GetLength(1);
int sampleRate = recording.SampleRate;
double herzPerBin = sampleRate / 2 / (double)rhzRowCount;
double scoreThreshold = (double?)double.Parse(configDict["EventThreshold"]) ?? 3.0;
int minimumFrequency = (int?)int.Parse(configDict["MinHz"]) ?? 850;
int dominantFrequency = (int?)int.Parse(configDict["DominantFrequency"]) ?? 1850;
// # The Limnodynastes call has three major peaks. The dominant peak is at 1850 or as set above.
// # The second and third peak are at equal gaps below. DominantFreq-gap and DominantFreq-(2*gap);
// # Set the gap in the Config file. Should typically be in range 880 to 970
int peakGapInHerz = (int?)int.Parse(configDict["PeakGap"]) ?? 470;
int F1AndF2Gap = (int)Math.Round(peakGapInHerz / herzPerBin);
//int F1AndF2Gap = 10; // 10 = number of freq bins
int F1AndF3Gap = 2 * F1AndF2Gap;
//int F1AndF3Gap = 20;
int hzBuffer = 250;
int bottomBin = 5;
int dominantBin = (int)Math.Round(dominantFrequency / herzPerBin);
int binBuffer = (int)Math.Round(hzBuffer / herzPerBin);;
int dominantBinMin = dominantBin - binBuffer;
int dominantBinMax = dominantBin + binBuffer;
// freqBin + rowID = binCount - 1;
// therefore: rowID = binCount - freqBin - 1;
int minRowID = rhzRowCount - dominantBinMax - 1;
int maxRowID = rhzRowCount - dominantBinMin - 1;
int bottomRow = rhzRowCount - bottomBin - 1;
var list = new List <Point>();
// loop through all spectra/columns of the hi-res spectrogram.
for (int c = 1; c < rhzColCount - 1; c++)
{
double maxAmplitude = -double.MaxValue;
int idOfRowWithMaxAmplitude = 0;
for (int r = minRowID; r <= bottomRow; r++)
{
if (spg[r, c] > maxAmplitude)
{
maxAmplitude = spg[r, c];
idOfRowWithMaxAmplitude = r;
}
}
if (idOfRowWithMaxAmplitude < minRowID)
{
continue;
}
if (idOfRowWithMaxAmplitude > maxRowID)
{
continue;
}
// want a spectral peak.
if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c - 1])
{
continue;
}
if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c + 1])
{
continue;
}
// peak should exceed thresold amplitude
if (spg[idOfRowWithMaxAmplitude, c] < 3.0)
{
continue;
}
// convert row ID to freq bin ID
int freqBinID = rhzRowCount - idOfRowWithMaxAmplitude - 1;
list.Add(new Point(c, freqBinID));
// we now have a list of potential hits for LimCon. This needs to be filtered.
// Console.WriteLine("Col {0}, Bin {1} ", c, freqBinID);
}
// DEBUG ONLY // ################################ TEMPORARY ################################
// superimpose point on RHZ HiRes spectrogram for debug purposes
bool drawOnHiResSpectrogram = true;
//string filePath = @"G:\SensorNetworks\Output\Frogs\TestOfHiResIndices-2016July\Test\Towsey.HiResIndices\SpectrogramImages\3mile_creek_dam_-_Herveys_Range_1076_248366_20130305_001700_30_0min.CombinedGreyScale.png";
var fileName = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name);
string filePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScale.png";
var debugImage = new FileInfo(filePath);
if (!debugImage.Exists)
{
drawOnHiResSpectrogram = false;
}
if (drawOnHiResSpectrogram)
{
// put red dot where max is
Bitmap bmp = new Bitmap(filePath);
foreach (Point point in list)
{
bmp.SetPixel(point.X + 70, 1911 - point.Y, Color.Red);
}
// mark off every tenth frequency bin
for (int r = 0; r < 26; r++)
{
bmp.SetPixel(68, 1911 - (r * 10), Color.Blue);
bmp.SetPixel(69, 1911 - (r * 10), Color.Blue);
}
// mark off upper bound and lower frequency bound
bmp.SetPixel(69, 1911 - dominantBinMin, Color.Lime);
bmp.SetPixel(69, 1911 - dominantBinMax, Color.Lime);
//bmp.SetPixel(69, 1911 - maxRowID, Color.Lime);
string opFilePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScaleAnnotated.png";
bmp.Save(opFilePath);
}
// END DEBUG ################################ TEMPORARY ################################
// now construct the standard decibel spectrogram WITHOUT noise removal, and look for LimConvex
// get frame parameters for the analysis
double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1);
int frameSize = rhzRowCount * 2;
int frameStep = frameSize; // this default = zero overlap
double frameDurationInSeconds = frameSize / (double)sampleRate;
double frameStepInSeconds = frameStep / (double)sampleRate;
double framesPerSec = 1 / frameStepInSeconds;
//var dspOutput = DSP_Frames.ExtractEnvelopeAndFFTs(recording, frameSize, frameStep);
//// Generate deciBel spectrogram
//double[,] deciBelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput.amplitudeSpectrogram, dspOutput.WindowPower, sampleRate, epsilon);
// i: Init SONOGRAM config
var sonoConfig = new SonogramConfig
{
SourceFName = recording.BaseName,
WindowSize = frameSize,
WindowOverlap = 0.0,
NoiseReductionType = NoiseReductionType.None,
};
// init sonogram
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// remove the DC row of the spectrogram
sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.Data.GetLength(0) - 1, sonogram.Data.GetLength(1) - 1);
//scores.Add(new Plot("Decibels", DataTools.NormaliseMatrixValues(dBArray), ActivityAndCover.DefaultActivityThresholdDb));
//scores.Add(new Plot("Active Frames", DataTools.Bool2Binary(activity.activeFrames), 0.0));
// convert spectral peaks to frequency
//var tuple_DecibelPeaks = SpectrogramTools.HistogramOfSpectralPeaks(deciBelSpectrogram);
//int[] peaksBins = tuple_DecibelPeaks.Item2;
//double[] freqPeaks = new double[peaksBins.Length];
//int binCount = sonogram.Data.GetLength(1);
//for (int i = 1; i < peaksBins.Length; i++) freqPeaks[i] = (lowerBinBound + peaksBins[i]) / (double)nyquistBin;
//scores.Add(new Plot("Max Frequency", freqPeaks, 0.0)); // location of peaks for spectral images
// create new list of LimCon hits in the standard spectrogram.
double timeSpanOfFrameInSeconds = frameSize / (double)sampleRate;
var newList = new List <int[]>();
int lastFrameID = sonogram.Data.GetLength(0) - 1;
int lastBinID = sonogram.Data.GetLength(1) - 1;
foreach (Point point in list)
{
double secondsFromStartOfSegment = (point.X * 0.1) + 0.05; // convert point.Y to center of time-block.
int framesFromStartOfSegment = (int)Math.Round(secondsFromStartOfSegment / timeSpanOfFrameInSeconds);
// location of max point is uncertain, so search in neighbourhood.
// NOTE: sonogram.data matrix is time*freqBin
double maxValue = -double.MaxValue;
int idOfTMax = framesFromStartOfSegment;
int idOfFMax = point.Y;
for (int deltaT = -4; deltaT <= 4; deltaT++)
{
for (int deltaF = -1; deltaF <= 1; deltaF++)
{
int newT = framesFromStartOfSegment + deltaT;
if (newT < 0)
{
newT = 0;
}
else if (newT > lastFrameID)
{
newT = lastFrameID;
}
double value = sonogram.Data[newT, point.Y + deltaF];
if (value > maxValue)
{
maxValue = value;
idOfTMax = framesFromStartOfSegment + deltaT;
idOfFMax = point.Y + deltaF;
}
}
}
// newList.Add(new Point(frameSpan, point.Y));
int[] array = new int[2];
array[0] = idOfTMax;
array[1] = idOfFMax;
newList.Add(array);
}
// Now obtain more of spectrogram to see if have peaks at two other places characteristic of Limnodynastes convex.
// In the D.Stewart CD, there are peaks close to:
//1. 1950 Hz
//2. 1460 hz
//3. 970 hz These are 490 Hz apart.
// For Limnodynastes convex, in the JCU recording, there are peaks close to:
//1. 1780 Hz
//2. 1330 hz
//3. 880 hz These are 450 Hz apart.
// So strategy is to look for three peaks separated by same amount and in the vicinity of the above,
// starting with highest power (the top peak) and working down to lowest power (bottom peak).
//We have found top/highest peak - now find the other two.
int secondDominantFrequency = 1380;
int secondDominantBin = (int)Math.Round(secondDominantFrequency / herzPerBin);
int thirdDominantFrequency = 900;
int thirdDominantBin = (int)Math.Round(thirdDominantFrequency / herzPerBin);
var acousticEvents = new List <AcousticEvent>();
int Tbuffer = 2;
// First extract a sub-matrix.
foreach (int[] array in newList)
{
// NOTE: sonogram.data matrix is time*freqBin
int Tframe = array[0];
int F1bin = array[1];
double[,] subMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - Tbuffer, 0, Tframe + Tbuffer, F1bin);
double F1power = subMatrix[Tbuffer, F1bin];
// convert to vector
var spectrum = MatrixTools.GetColumnAverages(subMatrix);
// use the following code to get estimate of background noise
double[,] powerMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - 3, 10, Tframe + 3, F1bin);
double averagePower = (MatrixTools.GetRowAverages(powerMatrix)).Average();
double score = F1power - averagePower;
// debug - checking what the spectrum looks like.
//for (int i = 0; i < 18; i++)
// spectrum[i] = -100.0;
//DataTools.writeBarGraph(spectrum);
// locate the peaks in lower frequency bands, F2 and F3
bool[] peaks = DataTools.GetPeaks(spectrum);
int F2bin = 0;
double F2power = -200.0; // dB
for (int i = -3; i <= 2; i++)
{
int bin = F1bin - F1AndF2Gap + i;
if ((peaks[bin]) && (F2power < subMatrix[1, bin]))
{
F2bin = bin;
F2power = subMatrix[1, bin];
}
}
if (F2bin == 0)
{
continue;
}
if (F2power == -200.0)
{
continue;
}
score += (F2power - averagePower);
int F3bin = 0;
double F3power = -200.0;
for (int i = -5; i <= 2; i++)
{
int bin = F1bin - F1AndF3Gap + i;
if ((peaks[bin]) && (F3power < subMatrix[1, bin]))
{
F3bin = bin;
F3power = subMatrix[1, bin];
}
}
if (F3bin == 0)
{
continue;
}
if (F3power == -200.0)
{
continue;
}
score += (F3power - averagePower);
score /= 3;
// ignore events where SNR < decibel threshold
if (score < scoreThreshold)
{
continue;
}
// ignore events with wrong power distribution. A good LimnoConvex call has strongest F1 power
if ((F3power > F1power) || (F2power > F1power))
{
continue;
}
//freq Bin ID must be converted back to Matrix row ID
// freqBin + rowID = binCount - 1;
// therefore: rowID = binCount - freqBin - 1;
minRowID = rhzRowCount - F1bin - 2;
maxRowID = rhzRowCount - F3bin - 1;
int F1RowID = rhzRowCount - F1bin - 1;
int F2RowID = rhzRowCount - F2bin - 1;
int F3RowID = rhzRowCount - F3bin - 1;
int maxfreq = dominantFrequency + hzBuffer;
int topBin = (int)Math.Round(maxfreq / herzPerBin);
int frameCount = 4;
double duration = frameCount * frameStepInSeconds;
double startTimeWrtSegment = (Tframe - 2) * frameStepInSeconds;
// Got to here so start initialising an acoustic event
var ae = new AcousticEvent(segmentStartOffset, startTimeWrtSegment, duration, minimumFrequency, maxfreq);
ae.SetTimeAndFreqScales(framesPerSec, herzPerBin);
//var ae = new AcousticEvent(oblong, recording.Nyquist, binCount, frameDurationInSeconds, frameStepInSeconds, frameCount);
//ae.StartOffset = TimeSpan.FromSeconds(Tframe * frameStepInSeconds);
var pointF1 = new Point(2, topBin - F1bin);
var pointF2 = new Point(2, topBin - F2bin);
var pointF3 = new Point(2, topBin - F3bin);
ae.Points = new List <Point>();
ae.Points.Add(pointF1);
ae.Points.Add(pointF2);
ae.Points.Add(pointF3);
//tried using HitElements but did not do what I wanted later on.
//ae.HitElements = new HashSet<Point>();
//ae.HitElements = new SortedSet<Point>();
//ae.HitElements.Add(pointF1);
//ae.HitElements.Add(pointF2);
//ae.HitElements.Add(pointF3);
ae.Score = score;
//ae.MinFreq = Math.Round((topBin - F3bin - 5) * herzPerBin);
//ae.MaxFreq = Math.Round(topBin * herzPerBin);
acousticEvents.Add(ae);
}
// now add in extra common info to the acoustic events
acousticEvents.ForEach(ae =>
{
ae.SpeciesName = configDict[AnalysisKeys.SpeciesName];
ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds;
ae.SegmentDurationSeconds = recording.Duration.TotalSeconds;
ae.Name = abbreviatedName;
ae.BorderColour = Color.Red;
ae.FileName = recording.BaseName;
});
double[] scores = new double[rhzColCount]; // predefinition of score array
double nomalisationConstant = scoreThreshold * 4; // four times the score threshold
double compressionFactor = rhzColCount / (double)sonogram.Data.GetLength(0);
foreach (AcousticEvent ae in acousticEvents)
{
ae.ScoreNormalised = ae.Score / nomalisationConstant;
if (ae.ScoreNormalised > 1.0)
{
ae.ScoreNormalised = 1.0;
}
int frameID = (int)Math.Round(ae.EventStartSeconds / frameDurationInSeconds);
int hiresFrameID = (int)Math.Floor(frameID * compressionFactor);
scores[hiresFrameID] = ae.ScoreNormalised;
}
var plot = new Plot(AnalysisName, scores, scoreThreshold);
// DEBUG ONLY ################################ TEMPORARY ################################
// Draw a standard spectrogram and mark of hites etc.
bool createStandardDebugSpectrogram = true;
var imageDir = new DirectoryInfo(outputDir.FullName + @"\SpectrogramImages");
if (!imageDir.Exists)
{
imageDir.Create();
}
if (createStandardDebugSpectrogram)
{
var fileName2 = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name);
string filePath2 = Path.Combine(imageDir.FullName, fileName + ".Spectrogram.png");
Bitmap sonoBmp = (Bitmap)sonogram.GetImage();
int height = sonoBmp.Height;
foreach (AcousticEvent ae in acousticEvents)
{
ae.DrawEvent(sonoBmp);
//g.DrawRectangle(pen, ob.ColumnLeft, ob.RowTop, ob.ColWidth-1, ob.RowWidth);
//ae.DrawPoint(sonoBmp, ae.HitElements.[0], Color.OrangeRed);
//ae.DrawPoint(sonoBmp, ae.HitElements[1], Color.Yellow);
//ae.DrawPoint(sonoBmp, ae.HitElements[2], Color.Green);
ae.DrawPoint(sonoBmp, ae.Points[0], Color.OrangeRed);
ae.DrawPoint(sonoBmp, ae.Points[1], Color.Yellow);
ae.DrawPoint(sonoBmp, ae.Points[2], Color.LimeGreen);
}
// draw the original hits on the standard sonogram
foreach (int[] array in newList)
{
sonoBmp.SetPixel(array[0], height - array[1], Color.Cyan);
}
// mark off every tenth frequency bin on the standard sonogram
for (int r = 0; r < 20; r++)
{
sonoBmp.SetPixel(0, height - (r * 10) - 1, Color.Blue);
sonoBmp.SetPixel(1, height - (r * 10) - 1, Color.Blue);
}
// mark off upper bound and lower frequency bound
sonoBmp.SetPixel(0, height - dominantBinMin, Color.Lime);
sonoBmp.SetPixel(0, height - dominantBinMax, Color.Lime);
sonoBmp.Save(filePath2);
}
// END DEBUG ################################ TEMPORARY ################################
return(new LimnodynastesConvexResults
{
Sonogram = sonogram,
Hits = null,
Plot = plot,
Events = acousticEvents,
RecordingDuration = recording.Duration,
});
} // Analysis()
/// <summary>
/// The CORE ANALYSIS METHOD.
/// </summary>
public static Tuple <BaseSonogram, double[, ], Plot, List <AcousticEvent>, TimeSpan> Analysis(FileInfo fiSegmentOfSourceFile, Dictionary <string, string> configDict, TimeSpan segmentStartOffset)
{
//set default values -
int frameLength = 1024;
if (configDict.ContainsKey(AnalysisKeys.FrameLength))
{
frameLength = int.Parse(configDict[AnalysisKeys.FrameLength]);
}
double windowOverlap = 0.0;
int minHz = int.Parse(configDict["MIN_HZ"]);
int minFormantgap = int.Parse(configDict["MIN_FORMANT_GAP"]);
int maxFormantgap = int.Parse(configDict["MAX_FORMANT_GAP"]);
double decibelThreshold = double.Parse(configDict["DECIBEL_THRESHOLD"]); //dB
double harmonicIntensityThreshold = double.Parse(configDict["INTENSITY_THRESHOLD"]); //in 0-1
double callDuration = double.Parse(configDict["CALL_DURATION"]); // seconds
AudioRecording recording = new AudioRecording(fiSegmentOfSourceFile.FullName);
//i: MAKE SONOGRAM
var sonoConfig = new SonogramConfig
{
SourceFName = recording.BaseName,
WindowSize = frameLength,
WindowOverlap = windowOverlap,
NoiseReductionType = SNR.KeyToNoiseReductionType("STANDARD"),
}; //default values config
TimeSpan tsRecordingtDuration = recording.Duration;
int sr = recording.SampleRate;
double freqBinWidth = sr / (double)sonoConfig.WindowSize;
double framesPerSecond = freqBinWidth;
//the Xcorrelation-FFT technique requires number of bins to scan to be power of 2.
//assuming sr=17640 and window=1024, then 64 bins span 1100 Hz above the min Hz level. i.e. 500 to 1600
//assuming sr=17640 and window=1024, then 128 bins span 2200 Hz above the min Hz level. i.e. 500 to 2700
int numberOfBins = 64;
int minBin = (int)Math.Round(minHz / freqBinWidth) + 1;
int maxbin = minBin + numberOfBins - 1;
int maxHz = (int)Math.Round(minHz + (numberOfBins * freqBinWidth));
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
int rowCount = sonogram.Data.GetLength(0);
int colCount = sonogram.Data.GetLength(1);
double[,] subMatrix = MatrixTools.Submatrix(sonogram.Data, 0, minBin, rowCount - 1, maxbin);
int callSpan = (int)Math.Round(callDuration * framesPerSecond);
//#############################################################################################################################################
//ii: DETECT HARMONICS
var results = CrossCorrelation.DetectHarmonicsInSonogramMatrix(subMatrix, decibelThreshold, callSpan);
double[] dBArray = results.Item1;
double[] intensity = results.Item2; //an array of periodicity scores
double[] periodicity = results.Item3;
//intensity = DataTools.filterMovingAverage(intensity, 3);
int noiseBound = (int)(100 / freqBinWidth); //ignore 0-100 hz - too much noise
double[] scoreArray = new double[intensity.Length];
for (int r = 0; r < rowCount; r++)
{
if (intensity[r] < harmonicIntensityThreshold)
{
continue;
}
//ignore locations with incorrect formant gap
double herzPeriod = periodicity[r] * freqBinWidth;
if (herzPeriod < minFormantgap || herzPeriod > maxFormantgap)
{
continue;
}
//find freq having max power and use info to adjust score.
//expect humans to have max < 1000 Hz
double[] spectrum = MatrixTools.GetRow(sonogram.Data, r);
for (int j = 0; j < noiseBound; j++)
{
spectrum[j] = 0.0;
}
int maxIndex = DataTools.GetMaxIndex(spectrum);
int freqWithMaxPower = (int)Math.Round(maxIndex * freqBinWidth);
double discount = 1.0;
if (freqWithMaxPower < 1200)
{
discount = 0.0;
}
if (intensity[r] > harmonicIntensityThreshold)
{
scoreArray[r] = intensity[r] * discount;
}
}
//transfer info to a hits matrix.
var hits = new double[rowCount, colCount];
double threshold = harmonicIntensityThreshold * 0.75; //reduced threshold for display of hits
for (int r = 0; r < rowCount; r++)
{
if (scoreArray[r] < threshold)
{
continue;
}
double herzPeriod = periodicity[r] * freqBinWidth;
for (int c = minBin; c < maxbin; c++)
{
//hits[r, c] = herzPeriod / (double)380; //divide by 380 to get a relativePeriod;
hits[r, c] = (herzPeriod - minFormantgap) / maxFormantgap; //to get a relativePeriod;
}
}
//iii: CONVERT TO ACOUSTIC EVENTS
double maxPossibleScore = 0.5;
int halfCallSpan = callSpan / 2;
var predictedEvents = new List <AcousticEvent>();
for (int i = 0; i < rowCount; i++)
{
//assume one score position per crow call
if (scoreArray[i] < 0.001)
{
continue;
}
double startTime = (i - halfCallSpan) / framesPerSecond;
AcousticEvent ev = new AcousticEvent(segmentStartOffset, startTime, callDuration, minHz, maxHz);
ev.SetTimeAndFreqScales(framesPerSecond, freqBinWidth);
ev.Score = scoreArray[i];
ev.ScoreNormalised = ev.Score / maxPossibleScore; // normalised to the user supplied threshold
//ev.Score_MaxPossible = maxPossibleScore;
predictedEvents.Add(ev);
} //for loop
Plot plot = new Plot("CROW", intensity, harmonicIntensityThreshold);
return(Tuple.Create(sonogram, hits, plot, predictedEvents, tsRecordingtDuration));
} //Analysis()
/// <summary>
/// Do your analysis. This method is called once per segment (typically one-minute segments).
/// </summary>
/// <param name="recording"></param>
/// <param name="configuration"></param>
/// <param name="segmentStartOffset"></param>
/// <param name="getSpectralIndexes"></param>
/// <param name="outputDirectory"></param>
/// <param name="imageWidth"></param>
/// <returns></returns>
public override RecognizerResults Recognize(AudioRecording recording, Config configuration, TimeSpan segmentStartOffset, Lazy <IndexCalculateResult[]> getSpectralIndexes, DirectoryInfo outputDirectory, int?imageWidth)
{
var recognizerConfig = new LitoriaNasutaConfig();
recognizerConfig.ReadConfigFile(configuration);
// BETTER TO SET THESE. IGNORE USER!
// this default framesize seems to work
const int frameSize = 1024;
const double windowOverlap = 0.0;
// i: MAKE SONOGRAM
var sonoConfig = new SonogramConfig
{
SourceFName = recording.BaseName,
WindowSize = frameSize,
WindowOverlap = windowOverlap,
// use the default HAMMING window
//WindowFunction = WindowFunctions.HANNING.ToString(),
//WindowFunction = WindowFunctions.NONE.ToString(),
// if do not use noise reduction can get a more sensitive recogniser.
//NoiseReductionType = NoiseReductionType.None
NoiseReductionType = NoiseReductionType.Standard,
NoiseReductionParameter = 0.0,
};
TimeSpan recordingDuration = recording.WavReader.Time;
int sr = recording.SampleRate;
double freqBinWidth = sr / (double)sonoConfig.WindowSize;
int minBin = (int)Math.Round(recognizerConfig.MinHz / freqBinWidth) + 1;
int maxBin = (int)Math.Round(recognizerConfig.MaxHz / freqBinWidth) + 1;
var decibelThreshold = 3.0;
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// ######################################################################
// ii: DO THE ANALYSIS AND RECOVER SCORES OR WHATEVER
int rowCount = sonogram.Data.GetLength(0);
double[] amplitudeArray = MatrixTools.GetRowAveragesOfSubmatrix(sonogram.Data, 0, minBin, rowCount - 1, maxBin);
//double[] topBand = MatrixTools.GetRowAveragesOfSubmatrix(sonogram.Data, 0, maxBin + 3, (rowCount - 1), maxBin + 9);
//double[] botBand = MatrixTools.GetRowAveragesOfSubmatrix(sonogram.Data, 0, minBin - 3, (rowCount - 1), minBin - 9);
// ii: DO THE ANALYSIS AND RECOVER SCORES OR WHATEVER
var acousticEvents = AcousticEvent.ConvertScoreArray2Events(
amplitudeArray,
recognizerConfig.MinHz,
recognizerConfig.MaxHz,
sonogram.FramesPerSecond,
freqBinWidth,
decibelThreshold,
recognizerConfig.MinDuration,
recognizerConfig.MaxDuration,
segmentStartOffset);
double[,] hits = null;
var prunedEvents = new List <AcousticEvent>();
acousticEvents.ForEach(ae =>
{
ae.SpeciesName = recognizerConfig.SpeciesName;
ae.SegmentDurationSeconds = recordingDuration.TotalSeconds;
ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds;
ae.Name = recognizerConfig.AbbreviatedSpeciesName;
});
var thresholdedPlot = new double[amplitudeArray.Length];
for (int x = 0; x < amplitudeArray.Length; x++)
{
if (amplitudeArray[x] > decibelThreshold)
{
thresholdedPlot[x] = amplitudeArray[x];
}
}
var maxDb = amplitudeArray.MaxOrDefault();
double[] normalisedScores;
double normalisedThreshold;
DataTools.Normalise(thresholdedPlot, decibelThreshold, out normalisedScores, out normalisedThreshold);
var text = string.Format($"{this.DisplayName} (Fullscale={maxDb:f1}dB)");
var plot = new Plot(text, normalisedScores, normalisedThreshold);
if (true)
{
// display a variety of debug score arrays
DataTools.Normalise(amplitudeArray, decibelThreshold, out normalisedScores, out normalisedThreshold);
var amplPlot = new Plot("Band amplitude", normalisedScores, normalisedThreshold);
var debugPlots = new List <Plot> {
plot, amplPlot
};
// NOTE: This DrawDebugImage() method can be over-written in this class.
var debugImage = DrawDebugImage(sonogram, acousticEvents, debugPlots, hits);
var debugPath = FilenameHelpers.AnalysisResultPath(outputDirectory, recording.BaseName, this.SpeciesName, "png", "DebugSpectrogram");
debugImage.Save(debugPath);
}
return(new RecognizerResults()
{
Sonogram = sonogram,
Hits = hits,
Plots = plot.AsList(),
Events = acousticEvents,
});
}
internal RecognizerResults Gruntwork(AudioRecording audioRecording, Config configuration, DirectoryInfo outputDirectory, TimeSpan segmentStartOffset)
{
double noiseReductionParameter = configuration.GetDoubleOrNull(AnalysisKeys.NoiseBgThreshold) ?? 0.1;
// make a spectrogram
var config = new SonogramConfig
{
WindowSize = 256,
NoiseReductionType = NoiseReductionType.Standard,
NoiseReductionParameter = noiseReductionParameter,
};
config.WindowOverlap = 0.0;
// now construct the standard decibel spectrogram WITH noise removal, and look for LimConvex
// get frame parameters for the analysis
var sonogram = (BaseSonogram) new SpectrogramStandard(config, audioRecording.WavReader);
// remove the DC column
var spg = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.Data.GetLength(0) - 1, sonogram.Data.GetLength(1) - 1);
int sampleRate = audioRecording.SampleRate;
int rowCount = spg.GetLength(0);
int colCount = spg.GetLength(1);
int frameSize = config.WindowSize;
int frameStep = frameSize; // this default = zero overlap
double frameStepInSeconds = frameStep / (double)sampleRate;
double framesPerSec = 1 / frameStepInSeconds;
// reading in variables from the config file
string speciesName = configuration[AnalysisKeys.SpeciesName] ?? "<no species>";
string abbreviatedSpeciesName = configuration[AnalysisKeys.AbbreviatedSpeciesName] ?? "<no.sp>";
int minHz = configuration.GetInt(AnalysisKeys.MinHz);
int maxHz = configuration.GetInt(AnalysisKeys.MaxHz);
// ## THREE THRESHOLDS ---- only one of these is given to user.
// minimum dB to register a dominant freq peak. After noise removal
double peakThresholdDb = 3.0;
// The threshold dB amplitude in the dominant freq bin required to yield an event
double eventThresholdDb = 6;
// minimum score for an acceptable event - that is when processing the score array.
double similarityThreshold = configuration.GetDoubleOrNull(AnalysisKeys.EventThreshold) ?? 0.2;
// IMPORTANT: The following frame durations assume a sampling rate = 22050 and window size of 256.
int minFrameWidth = 7;
int maxFrameWidth = 14;
double minDuration = (minFrameWidth - 1) * frameStepInSeconds;
double maxDuration = maxFrameWidth * frameStepInSeconds;
// Calculate Max Amplitude
int binMin = (int)Math.Round(minHz / sonogram.FBinWidth);
int binMax = (int)Math.Round(maxHz / sonogram.FBinWidth);
int[] dominantBins = new int[rowCount]; // predefinition of events max frequency
double[] scores = new double[rowCount]; // predefinition of score array
double[,] hits = new double[rowCount, colCount];
// loop through all spectra/rows of the spectrogram - NB: spg is rotated to vertical.
// mark the hits in hitMatrix
for (int s = 0; s < rowCount; s++)
{
double[] spectrum = MatrixTools.GetRow(spg, s);
double maxAmplitude = double.MinValue;
int maxId = 0;
// loop through bandwidth of L.onvex call and look for dominant frequency
for (int binID = 5; binID < binMax; binID++)
{
if (spectrum[binID] > maxAmplitude)
{
maxAmplitude = spectrum[binID];
maxId = binID;
}
}
if (maxId < binMin)
{
continue;
}
// peak should exceed thresold amplitude
if (spectrum[maxId] < peakThresholdDb)
{
continue;
}
scores[s] = maxAmplitude;
dominantBins[s] = maxId;
// Console.WriteLine("Col {0}, Bin {1} ", c, freqBinID);
} // loop through all spectra
// Find average amplitude
double[] amplitudeArray = MatrixTools.GetRowAveragesOfSubmatrix(
sonogram.Data,
0,
binMin,
rowCount - 1,
binMax);
var highPassFilteredSignal = DspFilters.SubtractBaseline(amplitudeArray, 7);
// We now have a list of potential hits for C. tinnula. This needs to be filtered.
var startEnds = new List <Point>();
Plot.FindStartsAndEndsOfScoreEvents(highPassFilteredSignal, eventThresholdDb, minFrameWidth, maxFrameWidth, out var prunedScores, out startEnds);
// High pass Filter
// loop through the score array and find beginning and end of potential events
var potentialEvents = new List <AcousticEvent>();
foreach (Point point in startEnds)
{
// get average of the dominant bin
int binSum = 0;
int binCount = 0;
int eventWidth = point.Y - point.X + 1;
for (int s = point.X; s <= point.Y; s++)
{
if (dominantBins[s] >= binMin)
{
binSum += dominantBins[s];
binCount++;
}
}
// find average dominant bin for the event
int avDominantBin = (int)Math.Round(binSum / (double)binCount);
int avDominantFreq = (int)(Math.Round(binSum / (double)binCount) * sonogram.FBinWidth);
// Get score for the event.
// Use a simple template for the honk and calculate cosine similarity to the template.
// Template has three dominant frequenices.
// minimum number of bins covering frequency bandwidth of C. tinnula call// minimum number of bins covering frequency bandwidth of L.convex call
int callBinWidth = 14;
var templates = GetCtinnulaTemplates(callBinWidth);
var eventMatrix = MatrixTools.Submatrix(spg, point.X, avDominantBin - callBinWidth + 2, point.Y, avDominantBin + 1);
double eventScore = GetEventScore(eventMatrix, templates);
// put hits into hits matrix
// put cosine score into the score array
for (int s = point.X; s <= point.Y; s++)
{
hits[s, avDominantBin] = 10;
prunedScores[s] = eventScore;
}
if (eventScore < similarityThreshold)
{
continue;
}
int topBinForEvent = avDominantBin + 2;
int bottomBinForEvent = topBinForEvent - callBinWidth;
double startTime = point.X * frameStepInSeconds;
double durationTime = eventWidth * frameStepInSeconds;
var newEvent = new AcousticEvent(segmentStartOffset, startTime, durationTime, minHz, maxHz);
newEvent.DominantFreq = avDominantFreq;
newEvent.Score = eventScore;
newEvent.SetTimeAndFreqScales(framesPerSec, sonogram.FBinWidth);
newEvent.Name = string.Empty; // remove name because it hides spectral content of the event.
potentialEvents.Add(newEvent);
}
// display the original score array
scores = DataTools.normalise(scores);
var debugPlot = new Plot(this.DisplayName, scores, similarityThreshold);
// DEBUG IMAGE this recognizer only. MUST set false for deployment.
bool displayDebugImage = MainEntry.InDEBUG;
if (displayDebugImage)
{
// display a variety of debug score arrays
DataTools.Normalise(amplitudeArray, eventThresholdDb, out var normalisedScores, out var normalisedThreshold);
var ampltdPlot = new Plot("Average amplitude", normalisedScores, normalisedThreshold);
DataTools.Normalise(highPassFilteredSignal, eventThresholdDb, out normalisedScores, out normalisedThreshold);
var demeanedPlot = new Plot("Hi Pass", normalisedScores, normalisedThreshold);
/*
* DataTools.Normalise(scores, eventThresholdDb, out normalisedScores, out normalisedThreshold);
* var ampltdPlot = new Plot("amplitude", normalisedScores, normalisedThreshold);
*
*
* DataTools.Normalise(lowPassFilteredSignal, decibelThreshold, out normalisedScores, out normalisedThreshold);
* var lowPassPlot = new Plot("Low Pass", normalisedScores, normalisedThreshold);
*/
var debugPlots = new List <Plot> {
ampltdPlot, demeanedPlot
};
Image debugImage = DisplayDebugImage(sonogram, potentialEvents, debugPlots, null);
var debugPath = outputDirectory.Combine(FilenameHelpers.AnalysisResultName(Path.GetFileNameWithoutExtension(audioRecording.BaseName), this.Identifier, "png", "DebugSpectrogram"));
debugImage.Save(debugPath.FullName);
}
// display the cosine similarity scores
var plot = new Plot(this.DisplayName, prunedScores, similarityThreshold);
var plots = new List <Plot> {
plot
};
// add names into the returned events
foreach (AcousticEvent ae in potentialEvents)
{
ae.Name = "speciesName"; // abbreviatedSpeciesName;
}
return(new RecognizerResults()
{
Events = potentialEvents,
Hits = hits,
Plots = plots,
Sonogram = sonogram,
});
}
// OTHER CONSTANTS
//private const string ImageViewer = @"C:\Windows\system32\mspaint.exe";
/// <summary>
/// Do your analysis. This method is called once per segment (typically one-minute segments).
/// </summary>
/// <param name="recording"></param>
/// <param name="configuration"></param>
/// <param name="segmentStartOffset"></param>
/// <param name="getSpectralIndexes"></param>
/// <param name="outputDirectory"></param>
/// <param name="imageWidth"></param>
/// <returns></returns>
public override RecognizerResults Recognize(AudioRecording recording, Config configuration, TimeSpan segmentStartOffset, Lazy <IndexCalculateResult[]> getSpectralIndexes, DirectoryInfo outputDirectory, int?imageWidth)
{
var recognizerConfig = new LitoriaWatjulumConfig();
recognizerConfig.ReadConfigFile(configuration);
//int maxOscilRate = (int)Math.Ceiling(1 / lwConfig.MinPeriod);
if (recording.WavReader.SampleRate != 22050)
{
throw new InvalidOperationException("Requires a 22050Hz file");
}
TimeSpan recordingDuration = recording.WavReader.Time;
// this default framesize seems to work
const int frameSize = 128;
double windowOverlap = 0.0;
// calculate the overlap instead
//double windowOverlap = Oscillations2012.CalculateRequiredFrameOverlap(
// recording.SampleRate,
// frameSize,
// maxOscilRate);
// i: MAKE SONOGRAM
var sonoConfig = new SonogramConfig
{
SourceFName = recording.BaseName,
//set default values - ignore those set by user
WindowSize = frameSize,
WindowOverlap = windowOverlap,
// the default window is HAMMING
//WindowFunction = WindowFunctions.HANNING.ToString(),
//WindowFunction = WindowFunctions.NONE.ToString(),
// if do not use noise reduction can get a more sensitive recogniser.
//NoiseReductionType = NoiseReductionType.NONE,
NoiseReductionType = SNR.KeyToNoiseReductionType("STANDARD"),
};
//#############################################################################################################################################
//DO THE ANALYSIS
var results = Analysis(recording, sonoConfig, recognizerConfig, MainEntry.InDEBUG, segmentStartOffset);
//######################################################################
if (results == null)
{
return(null); //nothing to process
}
var sonogram = results.Item1;
var hits = results.Item2;
var scoreArray = results.Item3;
var predictedEvents = results.Item4;
var debugImage = results.Item5;
// old way of creating a path:
//var debugPath = outputDirectory.Combine(FilenameHelpers.AnalysisResultName(Path.GetFileNameWithoutExtension(recording.FileName), SpeciesName, "png", "DebugSpectrogram"));
var debugPath = FilenameHelpers.AnalysisResultPath(outputDirectory, recording.BaseName, this.SpeciesName, "png", "DebugSpectrogram");
debugImage.Save(debugPath);
//#############################################################################################################################################
// Prune events here if required i.e. remove those below threshold score if this not already done. See other recognizers.
foreach (var ae in predictedEvents)
{
// add additional info
ae.Name = recognizerConfig.AbbreviatedSpeciesName;
ae.SpeciesName = recognizerConfig.SpeciesName;
ae.SegmentDurationSeconds = recordingDuration.TotalSeconds;
ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds;
}
// do a recognizer TEST.
if (false)
{
var testDir = new DirectoryInfo(outputDirectory.Parent.Parent.FullName);
TestTools.RecognizerScoresTest(recording.BaseName, testDir, recognizerConfig.AnalysisName, scoreArray);
AcousticEvent.TestToCompareEvents(recording.BaseName, testDir, recognizerConfig.AnalysisName, predictedEvents);
}
var plot = new Plot(this.DisplayName, scoreArray, recognizerConfig.EventThreshold);
return(new RecognizerResults()
{
Sonogram = sonogram,
Hits = hits,
Plots = plot.AsList(),
Events = predictedEvents,
});
}
