public void LinearFrequencyScale() { var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav"); var opFileStem = "BAC2_20071008"; var outputDir = this.outputDirectory; var outputImagePath = Path.Combine(outputDir.FullName, "LinearScaleSonogram.png"); var recording = new AudioRecording(recordingPath); // specfied linear scale int nyquist = 11025; int frameSize = 1024; int hertzInterval = 1000; var freqScale = new FrequencyScale(nyquist, frameSize, hertzInterval); var fst = freqScale.ScaleType; var sonoConfig = new SonogramConfig { WindowSize = freqScale.FinalBinCount * 2, WindowOverlap = 0.2, SourceFName = recording.BaseName, NoiseReductionType = NoiseReductionType.None, NoiseReductionParameter = 0.0, }; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // DO NOISE REDUCTION var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data); sonogram.Data = dataMatrix; sonogram.Configuration.WindowSize = freqScale.WindowSize; var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); image.Save(outputImagePath, ImageFormat.Png); // DO FILE EQUALITY TEST var stemOfExpectedFile = opFileStem + "_LinearScaleGridLineLocations.EXPECTED.json"; var stemOfActualFile = opFileStem + "_LinearScaleGridLineLocations.ACTUAL.json"; // Check that freqScale.GridLineLocations are correct var expectedFile1 = PathHelper.ResolveAsset("FrequencyScale\\" + stemOfExpectedFile); if (!expectedFile1.Exists) { LoggedConsole.WriteErrorLine("An EXPECTED results file does not exist. Test will fail!"); LoggedConsole.WriteErrorLine( $"If ACTUAL results file is correct, move it to dir `{PathHelper.TestResources}` and change its suffix to <.EXPECTED.json>"); } var resultFile1 = new FileInfo(Path.Combine(outputDir.FullName, stemOfActualFile)); Json.Serialise(resultFile1, freqScale.GridLineLocations); FileEqualityHelpers.TextFileEqual(expectedFile1, resultFile1); // Check that image dimensions are correct Assert.AreEqual(566, image.Height); Assert.AreEqual(1621, image.Width); }
public void LinearFrequencyScaleDefault() { // relative path because post-Build command transfers files to ...\\Work\GitHub\...\bin\Debug subfolder. var recordingPath = @"Recordings\BAC2_20071008-085040.wav"; var opFileStem = "BAC2_20071008"; var outputDir = this.outputDirectory; var outputImagePath = Path.Combine(outputDir.FullName, "DefaultLinearScaleSonogram.png"); var recording = new AudioRecording(recordingPath); // default linear scale var fst = FreqScaleType.Linear; var freqScale = new FrequencyScale(fst); var sonoConfig = new SonogramConfig { WindowSize = freqScale.FinalBinCount * 2, WindowOverlap = 0.2, SourceFName = recording.BaseName, NoiseReductionType = NoiseReductionType.None, NoiseReductionParameter = 0.0, }; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); sonogram.Configuration.WindowSize = freqScale.WindowSize; // DO NOISE REDUCTION var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data); sonogram.Data = dataMatrix; var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); image.Save(outputImagePath, ImageFormat.Png); // DO UNIT TESTING var stemOfExpectedFile = opFileStem + "_DefaultLinearScaleGridLineLocations.EXPECTED.json"; var stemOfActualFile = opFileStem + "_DefaultLinearScaleGridLineLocations.ACTUAL.json"; // Check that freqScale.GridLineLocations are correct var expectedFile1 = new FileInfo("FrequencyScale\\" + stemOfExpectedFile); if (!expectedFile1.Exists) { LoggedConsole.WriteErrorLine("An EXPECTED results file does not exist. Test will fail!"); LoggedConsole.WriteErrorLine("If ACTUAL results file is correct, move it to dir <...\\TestResources\\FrequencyScale> and change its suffix to <.EXPECTED.json>"); } var resultFile1 = new FileInfo(Path.Combine(outputDir.FullName, stemOfActualFile)); Json.Serialise(resultFile1, freqScale.GridLineLocations); FileEqualityHelpers.TextFileEqual(expectedFile1, resultFile1); // Check that image dimensions are correct Assert.AreEqual(310, image.Height); Assert.AreEqual(3247, image.Width); }
public void LinearFrequencyScale() { var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav"); var outputImagePath = this.outputDirectory.CombineFile("DefaultLinearScaleSonogram.png"); var recording = new AudioRecording(recordingPath); // specfied linear scale int nyquist = 11025; int frameSize = 1024; int hertzInterval = 1000; var freqScale = new FrequencyScale(nyquist, frameSize, hertzInterval); var fst = freqScale.ScaleType; var sonoConfig = new SonogramConfig { WindowSize = freqScale.FinalBinCount * 2, WindowOverlap = 0.2, SourceFName = recording.BaseName, NoiseReductionType = NoiseReductionType.None, NoiseReductionParameter = 0.0, }; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // DO NOISE REDUCTION var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data); sonogram.Data = dataMatrix; sonogram.Configuration.WindowSize = freqScale.WindowSize; var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst, freqScale.GridLineLocations); image.Save(outputImagePath); var expected = new[, ] { { 46, 1000 }, { 92, 2000 }, { 139, 3000 }, { 185, 4000 }, { 232, 5000 }, { 278, 6000 }, { 325, 7000 }, { 371, 8000 }, { 417, 9000 }, { 464, 10000 }, { 510, 11000 }, }; Assert.That.MatricesAreEqual(expected, freqScale.GridLineLocations); // Check that image dimensions are correct Assert.AreEqual(566, image.Height); Assert.AreEqual(1621, image.Width); }
public void LinearFrequencyScaleDefault() { var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav"); var outputImagePath = this.outputDirectory.CombineFile("DefaultLinearScaleSonogram.png"); var recording = new AudioRecording(recordingPath); // default linear scale var fst = FreqScaleType.Linear; var freqScale = new FrequencyScale(fst); var sonoConfig = new SonogramConfig { WindowSize = freqScale.FinalBinCount * 2, WindowOverlap = 0.2, SourceFName = recording.BaseName, NoiseReductionType = NoiseReductionType.None, NoiseReductionParameter = 0.0, }; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); sonogram.Configuration.WindowSize = freqScale.WindowSize; // DO NOISE REDUCTION var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data); sonogram.Data = dataMatrix; var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst, freqScale.GridLineLocations); image.Save(outputImagePath); // Check that freqScale.GridLineLocations are correct var expected = new[, ] { { 23, 1000 }, { 46, 2000 }, { 69, 3000 }, { 92, 4000 }, { 116, 5000 }, { 139, 6000 }, { 162, 7000 }, { 185, 8000 }, { 208, 9000 }, { 232, 10000 }, { 255, 11000 }, }; Assert.That.MatricesAreEqual(expected, freqScale.GridLineLocations); // Check that image dimensions are correct Assert.AreEqual(310, image.Height); Assert.AreEqual(3247, image.Width); }
/// <summary> /// METHOD TO CHECK IF SPECIFIED linear FREQ SCALE IS WORKING /// Check it on standard one minute recording. /// </summary> public static void TESTMETHOD_LinearFrequencyScale() { var recordingPath = @"C:\SensorNetworks\SoftwareTests\TestRecordings\BAC2_20071008-085040.wav"; var outputDir = @"C:\SensorNetworks\SoftwareTests\TestFrequencyScale".ToDirectoryInfo(); var expectedResultsDir = Path.Combine(outputDir.FullName, TestTools.ExpectedResultsDir).ToDirectoryInfo(); var outputImagePath = Path.Combine(outputDir.FullName, "linearScaleSonogram.png"); var opFileStem = "BAC2_20071008"; var recording = new AudioRecording(recordingPath); // specfied linear scale int nyquist = 11025; int frameSize = 1024; int hertzInterval = 1000; var freqScale = new FrequencyScale(nyquist, frameSize, hertzInterval); var fst = freqScale.ScaleType; var sonoConfig = new SonogramConfig { WindowSize = freqScale.FinalBinCount * 2, WindowOverlap = 0.2, SourceFName = recording.BaseName, //NoiseReductionType = NoiseReductionType.Standard, NoiseReductionType = NoiseReductionType.None, NoiseReductionParameter = 0.0, }; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // DO NOISE REDUCTION var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data); sonogram.Data = dataMatrix; sonogram.Configuration.WindowSize = freqScale.WindowSize; var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); image.Save(outputImagePath); // DO FILE EQUALITY TEST string testName = "testName"; var expectedTestFile = new FileInfo(Path.Combine(expectedResultsDir.FullName, "FrequencyLinearScaleTest.EXPECTED.json")); var resultFile = new FileInfo(Path.Combine(outputDir.FullName, opFileStem + "FrequencyLinearScaleTestResults.json")); Acoustics.Shared.Csv.Csv.WriteMatrixToCsv(resultFile, freqScale.GridLineLocations); TestTools.FileEqualityTest(testName, resultFile, expectedTestFile); LoggedConsole.WriteLine("Completed Linear Frequency Scale test"); Console.WriteLine("\n\n"); }
/// <summary> /// METHOD TO CHECK IF SPECIFIED MEL FREQ SCALE IS WORKING /// Check it on standard one minute recording. /// </summary> public static void TESTMETHOD_MelFrequencyScale() { var recordingPath = @"C:\SensorNetworks\SoftwareTests\TestRecordings\BAC2_20071008-085040.wav"; var outputDir = @"C:\SensorNetworks\SoftwareTests\TestFrequencyScale".ToDirectoryInfo(); var expectedResultsDir = Path.Combine(outputDir.FullName, TestTools.ExpectedResultsDir).ToDirectoryInfo(); var outputImagePath = Path.Combine(outputDir.FullName, "melScaleSonogram.png"); var opFileStem = "BAC2_20071008"; var recording = new AudioRecording(recordingPath); int nyquist = recording.Nyquist; int frameSize = 1024; int finalBinCount = 256; int hertzInterval = 1000; FreqScaleType scaleType = FreqScaleType.Mel; var freqScale = new FrequencyScale(scaleType, nyquist, frameSize, finalBinCount, hertzInterval); var fst = freqScale.ScaleType; var sonoConfig = new SonogramConfig { WindowSize = frameSize, WindowOverlap = 0.2, SourceFName = recording.BaseName, DoMelScale = (scaleType == FreqScaleType.Mel) ? true : false, MelBinCount = (scaleType == FreqScaleType.Mel) ? finalBinCount : frameSize / 2, //NoiseReductionType = NoiseReductionType.Standard, NoiseReductionType = NoiseReductionType.None, NoiseReductionParameter = 0.0, }; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // DRAW SPECTROGRAM var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); image.Save(outputImagePath); // DO FILE EQUALITY TEST string testName = "MelTest"; var expectedTestFile = new FileInfo(Path.Combine(expectedResultsDir.FullName, "MelFrequencyScaleTest.EXPECTED.json")); var resultFile = new FileInfo(Path.Combine(outputDir.FullName, opFileStem + "MelFrequencyLinearScaleTestResults.json")); Acoustics.Shared.Csv.Csv.WriteMatrixToCsv(resultFile, freqScale.GridLineLocations); TestTools.FileEqualityTest(testName, resultFile, expectedTestFile); LoggedConsole.WriteLine("Completed Mel Frequency Scale test"); Console.WriteLine("\n\n"); }
/// <summary> /// Generate a Spectrogram. /// </summary> /// <param name="bytes"> /// The bytes. /// </param> /// <returns> /// Spectrogram image. /// </returns> /// <exception cref="NotSupportedException"><c>NotSupportedException</c>.</exception> public Bitmap Spectrogram(byte[] bytes) { /* * 80 pixels per second is too quick for Silverlight. * we want to use 40 pixels per second (half - window size of 0) */ var sonogramConfig = new SonogramConfig { WindowOverlap = 0, // was 0.5 when ppms was 0.08 WindowSize = 512, DoSnr = false, // might save us some time generating spectrograms. }; Bitmap image; using (var audiorecording = new AudioRecording(bytes)) { // audiorecording.ConvertSampleRate22kHz(); // THIS METHOD CALL IS OBSOLETE if (audiorecording.SampleRate != 22050) { var msg = string.Format( "Must be able to convert audio to 22050hz. Audio has sample rate of {0}.", audiorecording.SampleRate); throw new NotSupportedException(msg); } using (var sonogram = new SpectrogramStandard(sonogramConfig, audiorecording.WavReader)) using (var img = sonogram.GetImage()) { image = new Bitmap(img); } } return(image); }
/// <summary> /// THE KEY ANALYSIS METHOD /// </summary> /// <param name="recording"> /// The segment Of Source File. /// </param> /// <param name="configDict"> /// The config Dict. /// </param> /// <param name="value"></param> /// <returns> /// The <see cref="LimnodynastesConvexResults"/>. /// </returns> internal static LimnodynastesConvexResults Analysis( Dictionary <string, double[, ]> dictionaryOfHiResSpectralIndices, AudioRecording recording, Dictionary <string, string> configDict, AnalysisSettings analysisSettings, SegmentSettingsBase segmentSettings) { // for Limnodynastes convex, in the D.Stewart CD, there are peaks close to: //1. 1950 Hz //2. 1460 hz //3. 970 hz These are 490 Hz apart. // for Limnodynastes convex, in the JCU recording, there are peaks close to: //1. 1780 Hz //2. 1330 hz //3. 880 hz These are 450 Hz apart. // So strategy is to look for three peaks separated by same amount and in the vicinity of the above, // starting with highest power (the top peak) and working down to lowest power (bottom peak). var outputDir = segmentSettings.SegmentOutputDirectory; TimeSpan segmentStartOffset = segmentSettings.SegmentStartOffset; //KeyValuePair<string, double[,]> kvp = dictionaryOfHiResSpectralIndices.First(); var spg = dictionaryOfHiResSpectralIndices["RHZ"]; int rhzRowCount = spg.GetLength(0); int rhzColCount = spg.GetLength(1); int sampleRate = recording.SampleRate; double herzPerBin = sampleRate / 2 / (double)rhzRowCount; double scoreThreshold = (double?)double.Parse(configDict["EventThreshold"]) ?? 3.0; int minimumFrequency = (int?)int.Parse(configDict["MinHz"]) ?? 850; int dominantFrequency = (int?)int.Parse(configDict["DominantFrequency"]) ?? 1850; // # The Limnodynastes call has three major peaks. The dominant peak is at 1850 or as set above. // # The second and third peak are at equal gaps below. DominantFreq-gap and DominantFreq-(2*gap); // # Set the gap in the Config file. Should typically be in range 880 to 970 int peakGapInHerz = (int?)int.Parse(configDict["PeakGap"]) ?? 470; int F1AndF2Gap = (int)Math.Round(peakGapInHerz / herzPerBin); //int F1AndF2Gap = 10; // 10 = number of freq bins int F1AndF3Gap = 2 * F1AndF2Gap; //int F1AndF3Gap = 20; int hzBuffer = 250; int bottomBin = 5; int dominantBin = (int)Math.Round(dominantFrequency / herzPerBin); int binBuffer = (int)Math.Round(hzBuffer / herzPerBin);; int dominantBinMin = dominantBin - binBuffer; int dominantBinMax = dominantBin + binBuffer; // freqBin + rowID = binCount - 1; // therefore: rowID = binCount - freqBin - 1; int minRowID = rhzRowCount - dominantBinMax - 1; int maxRowID = rhzRowCount - dominantBinMin - 1; int bottomRow = rhzRowCount - bottomBin - 1; var list = new List <Point>(); // loop through all spectra/columns of the hi-res spectrogram. for (int c = 1; c < rhzColCount - 1; c++) { double maxAmplitude = -double.MaxValue; int idOfRowWithMaxAmplitude = 0; for (int r = minRowID; r <= bottomRow; r++) { if (spg[r, c] > maxAmplitude) { maxAmplitude = spg[r, c]; idOfRowWithMaxAmplitude = r; } } if (idOfRowWithMaxAmplitude < minRowID) { continue; } if (idOfRowWithMaxAmplitude > maxRowID) { continue; } // want a spectral peak. if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c - 1]) { continue; } if (spg[idOfRowWithMaxAmplitude, c] < spg[idOfRowWithMaxAmplitude, c + 1]) { continue; } // peak should exceed thresold amplitude if (spg[idOfRowWithMaxAmplitude, c] < 3.0) { continue; } // convert row ID to freq bin ID int freqBinID = rhzRowCount - idOfRowWithMaxAmplitude - 1; list.Add(new Point(c, freqBinID)); // we now have a list of potential hits for LimCon. This needs to be filtered. // Console.WriteLine("Col {0}, Bin {1} ", c, freqBinID); } // DEBUG ONLY // ################################ TEMPORARY ################################ // superimpose point on RHZ HiRes spectrogram for debug purposes bool drawOnHiResSpectrogram = true; //string filePath = @"G:\SensorNetworks\Output\Frogs\TestOfHiResIndices-2016July\Test\Towsey.HiResIndices\SpectrogramImages\3mile_creek_dam_-_Herveys_Range_1076_248366_20130305_001700_30_0min.CombinedGreyScale.png"; var fileName = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name); string filePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScale.png"; var debugImage = new FileInfo(filePath); if (!debugImage.Exists) { drawOnHiResSpectrogram = false; } if (drawOnHiResSpectrogram) { // put red dot where max is Bitmap bmp = new Bitmap(filePath); foreach (Point point in list) { bmp.SetPixel(point.X + 70, 1911 - point.Y, Color.Red); } // mark off every tenth frequency bin for (int r = 0; r < 26; r++) { bmp.SetPixel(68, 1911 - (r * 10), Color.Blue); bmp.SetPixel(69, 1911 - (r * 10), Color.Blue); } // mark off upper bound and lower frequency bound bmp.SetPixel(69, 1911 - dominantBinMin, Color.Lime); bmp.SetPixel(69, 1911 - dominantBinMax, Color.Lime); //bmp.SetPixel(69, 1911 - maxRowID, Color.Lime); string opFilePath = outputDir.FullName + @"\SpectrogramImages\" + fileName + ".CombinedGreyScaleAnnotated.png"; bmp.Save(opFilePath); } // END DEBUG ################################ TEMPORARY ################################ // now construct the standard decibel spectrogram WITHOUT noise removal, and look for LimConvex // get frame parameters for the analysis double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1); int frameSize = rhzRowCount * 2; int frameStep = frameSize; // this default = zero overlap double frameDurationInSeconds = frameSize / (double)sampleRate; double frameStepInSeconds = frameStep / (double)sampleRate; double framesPerSec = 1 / frameStepInSeconds; //var dspOutput = DSP_Frames.ExtractEnvelopeAndFFTs(recording, frameSize, frameStep); //// Generate deciBel spectrogram //double[,] deciBelSpectrogram = MFCCStuff.DecibelSpectra(dspOutput.amplitudeSpectrogram, dspOutput.WindowPower, sampleRate, epsilon); // i: Init SONOGRAM config var sonoConfig = new SonogramConfig { SourceFName = recording.BaseName, WindowSize = frameSize, WindowOverlap = 0.0, NoiseReductionType = NoiseReductionType.None, }; // init sonogram BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // remove the DC row of the spectrogram sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.Data.GetLength(0) - 1, sonogram.Data.GetLength(1) - 1); //scores.Add(new Plot("Decibels", DataTools.NormaliseMatrixValues(dBArray), ActivityAndCover.DefaultActivityThresholdDb)); //scores.Add(new Plot("Active Frames", DataTools.Bool2Binary(activity.activeFrames), 0.0)); // convert spectral peaks to frequency //var tuple_DecibelPeaks = SpectrogramTools.HistogramOfSpectralPeaks(deciBelSpectrogram); //int[] peaksBins = tuple_DecibelPeaks.Item2; //double[] freqPeaks = new double[peaksBins.Length]; //int binCount = sonogram.Data.GetLength(1); //for (int i = 1; i < peaksBins.Length; i++) freqPeaks[i] = (lowerBinBound + peaksBins[i]) / (double)nyquistBin; //scores.Add(new Plot("Max Frequency", freqPeaks, 0.0)); // location of peaks for spectral images // create new list of LimCon hits in the standard spectrogram. double timeSpanOfFrameInSeconds = frameSize / (double)sampleRate; var newList = new List <int[]>(); int lastFrameID = sonogram.Data.GetLength(0) - 1; int lastBinID = sonogram.Data.GetLength(1) - 1; foreach (Point point in list) { double secondsFromStartOfSegment = (point.X * 0.1) + 0.05; // convert point.Y to center of time-block. int framesFromStartOfSegment = (int)Math.Round(secondsFromStartOfSegment / timeSpanOfFrameInSeconds); // location of max point is uncertain, so search in neighbourhood. // NOTE: sonogram.data matrix is time*freqBin double maxValue = -double.MaxValue; int idOfTMax = framesFromStartOfSegment; int idOfFMax = point.Y; for (int deltaT = -4; deltaT <= 4; deltaT++) { for (int deltaF = -1; deltaF <= 1; deltaF++) { int newT = framesFromStartOfSegment + deltaT; if (newT < 0) { newT = 0; } else if (newT > lastFrameID) { newT = lastFrameID; } double value = sonogram.Data[newT, point.Y + deltaF]; if (value > maxValue) { maxValue = value; idOfTMax = framesFromStartOfSegment + deltaT; idOfFMax = point.Y + deltaF; } } } // newList.Add(new Point(frameSpan, point.Y)); int[] array = new int[2]; array[0] = idOfTMax; array[1] = idOfFMax; newList.Add(array); } // Now obtain more of spectrogram to see if have peaks at two other places characteristic of Limnodynastes convex. // In the D.Stewart CD, there are peaks close to: //1. 1950 Hz //2. 1460 hz //3. 970 hz These are 490 Hz apart. // For Limnodynastes convex, in the JCU recording, there are peaks close to: //1. 1780 Hz //2. 1330 hz //3. 880 hz These are 450 Hz apart. // So strategy is to look for three peaks separated by same amount and in the vicinity of the above, // starting with highest power (the top peak) and working down to lowest power (bottom peak). //We have found top/highest peak - now find the other two. int secondDominantFrequency = 1380; int secondDominantBin = (int)Math.Round(secondDominantFrequency / herzPerBin); int thirdDominantFrequency = 900; int thirdDominantBin = (int)Math.Round(thirdDominantFrequency / herzPerBin); var acousticEvents = new List <AcousticEvent>(); int Tbuffer = 2; // First extract a sub-matrix. foreach (int[] array in newList) { // NOTE: sonogram.data matrix is time*freqBin int Tframe = array[0]; int F1bin = array[1]; double[,] subMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - Tbuffer, 0, Tframe + Tbuffer, F1bin); double F1power = subMatrix[Tbuffer, F1bin]; // convert to vector var spectrum = MatrixTools.GetColumnAverages(subMatrix); // use the following code to get estimate of background noise double[,] powerMatrix = MatrixTools.Submatrix(sonogram.Data, Tframe - 3, 10, Tframe + 3, F1bin); double averagePower = (MatrixTools.GetRowAverages(powerMatrix)).Average(); double score = F1power - averagePower; // debug - checking what the spectrum looks like. //for (int i = 0; i < 18; i++) // spectrum[i] = -100.0; //DataTools.writeBarGraph(spectrum); // locate the peaks in lower frequency bands, F2 and F3 bool[] peaks = DataTools.GetPeaks(spectrum); int F2bin = 0; double F2power = -200.0; // dB for (int i = -3; i <= 2; i++) { int bin = F1bin - F1AndF2Gap + i; if ((peaks[bin]) && (F2power < subMatrix[1, bin])) { F2bin = bin; F2power = subMatrix[1, bin]; } } if (F2bin == 0) { continue; } if (F2power == -200.0) { continue; } score += (F2power - averagePower); int F3bin = 0; double F3power = -200.0; for (int i = -5; i <= 2; i++) { int bin = F1bin - F1AndF3Gap + i; if ((peaks[bin]) && (F3power < subMatrix[1, bin])) { F3bin = bin; F3power = subMatrix[1, bin]; } } if (F3bin == 0) { continue; } if (F3power == -200.0) { continue; } score += (F3power - averagePower); score /= 3; // ignore events where SNR < decibel threshold if (score < scoreThreshold) { continue; } // ignore events with wrong power distribution. A good LimnoConvex call has strongest F1 power if ((F3power > F1power) || (F2power > F1power)) { continue; } //freq Bin ID must be converted back to Matrix row ID // freqBin + rowID = binCount - 1; // therefore: rowID = binCount - freqBin - 1; minRowID = rhzRowCount - F1bin - 2; maxRowID = rhzRowCount - F3bin - 1; int F1RowID = rhzRowCount - F1bin - 1; int F2RowID = rhzRowCount - F2bin - 1; int F3RowID = rhzRowCount - F3bin - 1; int maxfreq = dominantFrequency + hzBuffer; int topBin = (int)Math.Round(maxfreq / herzPerBin); int frameCount = 4; double duration = frameCount * frameStepInSeconds; double startTimeWrtSegment = (Tframe - 2) * frameStepInSeconds; // Got to here so start initialising an acoustic event var ae = new AcousticEvent(segmentStartOffset, startTimeWrtSegment, duration, minimumFrequency, maxfreq); ae.SetTimeAndFreqScales(framesPerSec, herzPerBin); //var ae = new AcousticEvent(oblong, recording.Nyquist, binCount, frameDurationInSeconds, frameStepInSeconds, frameCount); //ae.StartOffset = TimeSpan.FromSeconds(Tframe * frameStepInSeconds); var pointF1 = new Point(2, topBin - F1bin); var pointF2 = new Point(2, topBin - F2bin); var pointF3 = new Point(2, topBin - F3bin); ae.Points = new List <Point>(); ae.Points.Add(pointF1); ae.Points.Add(pointF2); ae.Points.Add(pointF3); //tried using HitElements but did not do what I wanted later on. //ae.HitElements = new HashSet<Point>(); //ae.HitElements = new SortedSet<Point>(); //ae.HitElements.Add(pointF1); //ae.HitElements.Add(pointF2); //ae.HitElements.Add(pointF3); ae.Score = score; //ae.MinFreq = Math.Round((topBin - F3bin - 5) * herzPerBin); //ae.MaxFreq = Math.Round(topBin * herzPerBin); acousticEvents.Add(ae); } // now add in extra common info to the acoustic events acousticEvents.ForEach(ae => { ae.SpeciesName = configDict[AnalysisKeys.SpeciesName]; ae.SegmentStartSeconds = segmentStartOffset.TotalSeconds; ae.SegmentDurationSeconds = recording.Duration.TotalSeconds; ae.Name = abbreviatedName; ae.BorderColour = Color.Red; ae.FileName = recording.BaseName; }); double[] scores = new double[rhzColCount]; // predefinition of score array double nomalisationConstant = scoreThreshold * 4; // four times the score threshold double compressionFactor = rhzColCount / (double)sonogram.Data.GetLength(0); foreach (AcousticEvent ae in acousticEvents) { ae.ScoreNormalised = ae.Score / nomalisationConstant; if (ae.ScoreNormalised > 1.0) { ae.ScoreNormalised = 1.0; } int frameID = (int)Math.Round(ae.EventStartSeconds / frameDurationInSeconds); int hiresFrameID = (int)Math.Floor(frameID * compressionFactor); scores[hiresFrameID] = ae.ScoreNormalised; } var plot = new Plot(AnalysisName, scores, scoreThreshold); // DEBUG ONLY ################################ TEMPORARY ################################ // Draw a standard spectrogram and mark of hites etc. bool createStandardDebugSpectrogram = true; var imageDir = new DirectoryInfo(outputDir.FullName + @"\SpectrogramImages"); if (!imageDir.Exists) { imageDir.Create(); } if (createStandardDebugSpectrogram) { var fileName2 = Path.GetFileNameWithoutExtension(segmentSettings.SegmentAudioFile.Name); string filePath2 = Path.Combine(imageDir.FullName, fileName + ".Spectrogram.png"); Bitmap sonoBmp = (Bitmap)sonogram.GetImage(); int height = sonoBmp.Height; foreach (AcousticEvent ae in acousticEvents) { ae.DrawEvent(sonoBmp); //g.DrawRectangle(pen, ob.ColumnLeft, ob.RowTop, ob.ColWidth-1, ob.RowWidth); //ae.DrawPoint(sonoBmp, ae.HitElements.[0], Color.OrangeRed); //ae.DrawPoint(sonoBmp, ae.HitElements[1], Color.Yellow); //ae.DrawPoint(sonoBmp, ae.HitElements[2], Color.Green); ae.DrawPoint(sonoBmp, ae.Points[0], Color.OrangeRed); ae.DrawPoint(sonoBmp, ae.Points[1], Color.Yellow); ae.DrawPoint(sonoBmp, ae.Points[2], Color.LimeGreen); } // draw the original hits on the standard sonogram foreach (int[] array in newList) { sonoBmp.SetPixel(array[0], height - array[1], Color.Cyan); } // mark off every tenth frequency bin on the standard sonogram for (int r = 0; r < 20; r++) { sonoBmp.SetPixel(0, height - (r * 10) - 1, Color.Blue); sonoBmp.SetPixel(1, height - (r * 10) - 1, Color.Blue); } // mark off upper bound and lower frequency bound sonoBmp.SetPixel(0, height - dominantBinMin, Color.Lime); sonoBmp.SetPixel(0, height - dominantBinMax, Color.Lime); sonoBmp.Save(filePath2); } // END DEBUG ################################ TEMPORARY ################################ return(new LimnodynastesConvexResults { Sonogram = sonogram, Hits = null, Plot = plot, Events = acousticEvents, RecordingDuration = recording.Duration, }); } // Analysis()
public void TestPcaWhitening() { var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav"); var outputDir = this.outputDirectory; var outputImagePath = Path.Combine(outputDir.FullName, "ReconstrcutedSpectrogram.png"); var fst = FreqScaleType.Linear; var freqScale = new FrequencyScale(fst); var recording = new AudioRecording(recordingPath); var sonoConfig = new SonogramConfig { WindowSize = freqScale.FinalBinCount * 2, WindowOverlap = 0.2, SourceFName = recording.BaseName, NoiseReductionType = NoiseReductionType.None, NoiseReductionParameter = 0.0, }; // GENERATE AMPLITUDE SPECTROGRAM var spectrogram = new AmplitudeSonogram(sonoConfig, recording.WavReader); spectrogram.Configuration.WindowSize = freqScale.WindowSize; // DO RMS NORMALIZATION spectrogram.Data = SNR.RmsNormalization(spectrogram.Data); // CONVERT NORMALIZED AMPLITUDE SPECTROGRAM TO dB SPECTROGRAM var sonogram = new SpectrogramStandard(spectrogram); // DO NOISE REDUCTION var dataMatrix = PcaWhitening.NoiseReduction(sonogram.Data); sonogram.Data = dataMatrix; // Do Patch Sampling int rows = sonogram.Data.GetLength(0); int columns = sonogram.Data.GetLength(1); int patchWidth = columns; int patchHeight = 1; int numberOfPatches = (rows / patchHeight) * (columns / patchWidth); var sequentialPatches = PatchSampling.GetPatches(sonogram.Data, patchWidth, patchHeight, numberOfPatches, PatchSampling.SamplingMethod.Sequential); double[,] sequentialPatchMatrix = sequentialPatches.ToMatrix(); // DO PCA WHITENING var whitenedSpectrogram = PcaWhitening.Whitening(sequentialPatchMatrix); // reconstructing the spectrogram from sequential patches and the projection matrix obtained from random patches var projectionMatrix = whitenedSpectrogram.ProjectionMatrix;//whitenedSpectrogram.projectionMatrix; var eigenVectors = whitenedSpectrogram.EigenVectors; var numComponents = whitenedSpectrogram.Components; double[,] reconstructedSpec = PcaWhitening.ReconstructSpectrogram(projectionMatrix, sequentialPatchMatrix, eigenVectors, numComponents); sonogram.Data = PatchSampling.ConvertPatches(reconstructedSpec, patchWidth, patchHeight, columns); var reconstructedSpecImage = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "RECONSTRUCTEDSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); reconstructedSpecImage.Save(outputImagePath, ImageFormat.Png); // DO UNIT TESTING Assert.AreEqual(spectrogram.Data.GetLength(0), sonogram.Data.GetLength(0)); Assert.AreEqual(spectrogram.Data.GetLength(1), sonogram.Data.GetLength(1)); }
public static void Main(Arguments arguments) { //1. set up the necessary files //DirectoryInfo diSource = arguments.Source.Directory; FileInfo fiSourceRecording = arguments.Source; FileInfo fiConfig = arguments.Config.ToFileInfo(); FileInfo fiImage = arguments.Output.ToFileInfo(); fiImage.CreateParentDirectories(); string title = "# CREATE FOUR (4) SONOGRAMS FROM AUDIO RECORDING"; string date = "# DATE AND TIME: " + DateTime.Now; LoggedConsole.WriteLine(title); LoggedConsole.WriteLine(date); LoggedConsole.WriteLine("# Input audio file: " + fiSourceRecording.Name); LoggedConsole.WriteLine("# Output image file: " + fiImage); //2. get the config dictionary Config configuration = ConfigFile.Deserialize(fiConfig); //below three lines are examples of retrieving info from Config config //string analysisIdentifier = configuration[AnalysisKeys.AnalysisName]; //bool saveIntermediateWavFiles = (bool?)configuration[AnalysisKeys.SaveIntermediateWavFiles] ?? false; //scoreThreshold = (double?)configuration[AnalysisKeys.EventThreshold] ?? scoreThreshold; //3 transfer conogram parameters to a dictionary to be passed around var configDict = new Dictionary <string, string>(); // #Resample rate must be 2 X the desired Nyquist. Default is that of recording. configDict["ResampleRate"] = (configuration.GetIntOrNull(AnalysisKeys.ResampleRate) ?? 17640).ToString(); configDict["FrameLength"] = configuration[AnalysisKeys.FrameLength] ?? "512"; int frameSize = configuration.GetIntOrNull(AnalysisKeys.FrameLength) ?? 512; // #Frame Overlap as fraction: default=0.0 configDict["FrameOverlap"] = configuration[AnalysisKeys.FrameOverlap] ?? "0.0"; double windowOverlap = configuration.GetDoubleOrNull(AnalysisKeys.FrameOverlap) ?? 0.0; // #MinHz: 500 // #MaxHz: 3500 // #NOISE REDUCTION PARAMETERS configDict["DoNoiseReduction"] = configuration["DoNoiseReduction"] ?? "true"; configDict["BgNoiseThreshold"] = configuration["BgNoiseThreshold"] ?? "3.0"; configDict["ADD_AXES"] = configuration["ADD_AXES"] ?? "true"; configDict["AddSegmentationTrack"] = configuration["AddSegmentationTrack"] ?? "true"; // 3: GET RECORDING var startOffsetMins = TimeSpan.Zero; var endOffsetMins = TimeSpan.Zero; FileInfo fiOutputSegment = fiSourceRecording; if (!(startOffsetMins == TimeSpan.Zero && endOffsetMins == TimeSpan.Zero)) { var buffer = new TimeSpan(0, 0, 0); fiOutputSegment = new FileInfo(Path.Combine(fiImage.DirectoryName, "tempWavFile.wav")); //This method extracts segment and saves to disk at the location fiOutputSegment. var resampleRate = configuration.GetIntOrNull(AnalysisKeys.ResampleRate) ?? AppConfigHelper.DefaultTargetSampleRate; AudioRecording.ExtractSegment(fiSourceRecording, startOffsetMins, endOffsetMins, buffer, resampleRate, fiOutputSegment); } var recording = new AudioRecording(fiOutputSegment.FullName); // EXTRACT ENVELOPE and SPECTROGRAM var dspOutput = DSP_Frames.ExtractEnvelopeAndFfts(recording, frameSize, windowOverlap); // average absolute value over the minute recording ////double[] avAbsolute = dspOutput.Average; // (A) ################################## EXTRACT INDICES FROM THE SIGNAL WAVEFORM ################################## // var wavDuration = TimeSpan.FromSeconds(recording.WavReader.Time.TotalSeconds); // double totalSeconds = wavDuration.TotalSeconds; // double[] signalEnvelope = dspOutput.Envelope; // double avSignalEnvelope = signalEnvelope.Average(); // double[] frameEnergy = dspOutput.FrameEnergy; // double highAmplIndex = dspOutput.HighAmplitudeCount / totalSeconds; // double binWidth = dspOutput.BinWidth; // int nyquistBin = dspOutput.NyquistBin; // dspOutput.WindowPower, // dspOutput.FreqBinWidth int nyquistFreq = dspOutput.NyquistFreq; double epsilon = recording.Epsilon; // i: prepare amplitude spectrogram double[,] amplitudeSpectrogramData = dspOutput.AmplitudeSpectrogram; // get amplitude spectrogram. var image1 = ImageTools.DrawReversedMatrix(MatrixTools.MatrixRotate90Anticlockwise(amplitudeSpectrogramData)); // ii: prepare decibel spectrogram prior to noise removal double[,] decibelSpectrogramdata = MFCCStuff.DecibelSpectra(dspOutput.AmplitudeSpectrogram, dspOutput.WindowPower, recording.SampleRate, epsilon); decibelSpectrogramdata = MatrixTools.NormaliseMatrixValues(decibelSpectrogramdata); var image2 = ImageTools.DrawReversedMatrix(MatrixTools.MatrixRotate90Anticlockwise(decibelSpectrogramdata)); // iii: Calculate background noise spectrum in decibels // Calculate noise value for each freq bin. double sdCount = 0.0; // number of SDs above the mean for noise removal var decibelProfile = NoiseProfile.CalculateModalNoiseProfile(decibelSpectrogramdata, sdCount); // DataTools.writeBarGraph(dBProfile.NoiseMode); // iv: Prepare noise reduced spectrogram decibelSpectrogramdata = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogramdata, decibelProfile.NoiseThresholds); //double dBThreshold = 1.0; // SPECTRAL dB THRESHOLD for smoothing background //decibelSpectrogramdata = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogramdata, dBThreshold); var image3 = ImageTools.DrawReversedMatrix(MatrixTools.MatrixRotate90Anticlockwise(decibelSpectrogramdata)); // prepare new sonogram config and draw second image going down different code pathway var config = new SonogramConfig { MinFreqBand = 0, MaxFreqBand = 10000, NoiseReductionType = SNR.KeyToNoiseReductionType("Standard"), NoiseReductionParameter = 1.0, WindowSize = frameSize, WindowOverlap = windowOverlap, }; //var mfccConfig = new MfccConfiguration(config); int bandCount = config.mfccConfig.FilterbankCount; bool doMelScale = config.mfccConfig.DoMelScale; int ccCount = config.mfccConfig.CcCount; int fftBins = config.FreqBinCount; //number of Hz bands = 2^N +1 because includes the DC band int minHz = config.MinFreqBand ?? 0; int maxHz = config.MaxFreqBand ?? nyquistFreq; var standardSonogram = new SpectrogramStandard(config, recording.WavReader); var image4 = standardSonogram.GetImage(); // TODO next line crashes - does not produce cepstral sonogram. //SpectrogramCepstral cepSng = new SpectrogramCepstral(config, recording.WavReader); //Image image5 = cepSng.GetImage(); //var mti = SpectrogramTools.Sonogram2MultiTrackImage(sonogram, configDict); //var image = mti.GetImage(); //Image image = SpectrogramTools.Matrix2SonogramImage(deciBelSpectrogram, config); //Image image = SpectrogramTools.Audio2SonogramImage(FileInfo fiAudio, Dictionary<string, string> configDict); //prepare sonogram images var protoImage6 = new Image_MultiTrack(standardSonogram.GetImage(doHighlightSubband: false, add1KHzLines: true, doMelScale: false)); protoImage6.AddTrack(ImageTrack.GetTimeTrack(standardSonogram.Duration, standardSonogram.FramesPerSecond)); protoImage6.AddTrack(ImageTrack.GetWavEnvelopeTrack(recording, protoImage6.SonogramImage.Width)); protoImage6.AddTrack(ImageTrack.GetSegmentationTrack(standardSonogram)); var image6 = protoImage6.GetImage(); var list = new List <Image <Rgb24> >(); list.Add(image1); // amplitude spectrogram list.Add(image2); // decibel spectrogram before noise removal list.Add(image3); // decibel spectrogram after noise removal list.Add(image4); // second version of noise reduced spectrogram //list.Add(image5); // ceptral sonogram list.Add(image6.CloneAs <Rgb24>()); // multitrack image Image finalImage = ImageTools.CombineImagesVertically(list); finalImage.Save(fiImage.FullName); ////2: NOISE REMOVAL //double[,] originalSg = sonogram.Data; //double[,] mnr = sonogram.Data; //mnr = ImageTools.WienerFilter(mnr, 3); //double backgroundThreshold = 4.0; //SETS MIN DECIBEL BOUND //var output = SNR.NoiseReduce(mnr, NoiseReductionType.STANDARD, backgroundThreshold); //double ConfigRange = 70; //sets the the max dB //mnr = SNR.SetConfigRange(output.Item1, 0.0, ConfigRange); ////3: Spectral tracks sonogram //byte[,] binary = MatrixTools.IdentifySpectralRidges(mnr); //binary = MatrixTools.ThresholdBinarySpectrum(binary, mnr, 10); //binary = MatrixTools.RemoveOrphanOnesInBinaryMatrix(binary); ////binary = MatrixTools.PickOutLines(binary); //syntactic approach //sonogram.SetBinarySpectrum(binary); ////sonogram.Data = SNR.SpectralRidges2Intensity(binary, originalSg); //image = new Image_MultiTrack(sonogram.GetImage(doHighlightSubband, false)); //image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond)); //image.AddTrack(ImageTrack.GetWavEnvelopeTrack(recording, image.sonogramImage.Width)); //image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); //fn = outputFolder + wavFileName + "_tracks.png"; //image.Save(fn); //LoggedConsole.WriteLine("Spectral tracks sonogram to file: " + fn); //3: prepare image of spectral peaks sonogram //sonogram.Data = SNR.NoiseReduce_Peaks(originalSg, dynamicRange); //image = new Image_MultiTrack(sonogram.GetImage(doHighlightSubband, add1kHzLines)); //image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration)); //image.AddTrack(ImageTrack.GetWavEnvelopeTrack(recording, image.Image.Width)); //image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); //fn = outputFolder + wavFileName + "_peaks.png"; //image.Save(fn); //LoggedConsole.WriteLine("Spectral peaks sonogram to file: " + fn); //4: Sobel approach //sonogram.Data = SNR.NoiseReduce_Sobel(originalSg, dynamicRange); //image = new Image_MultiTrack(sonogram.GetImage(doHighlightSubband, add1kHzLines)); //image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration)); //image.AddTrack(ImageTrack.GetWavEnvelopeTrack(recording, image.Image.Width)); //image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); //fn = outputFolder + wavFileName + "_sobel.png"; //image.Save(fn); //LoggedConsole.WriteLine("Sobel sonogram to file: " + fn); // I1.txt contains the sonogram matrix produced by matlab //string matlabFile = @"C:\SensorNetworks\Software\AudioAnalysis\AED\Test\matlab\GParrots_JB2_20090607-173000.wav_minute_3\I1.txt"; //double[,] matlabMatrix = Util.fileToMatrix(matlabFile, 256, 5166); //LoggedConsole.WriteLine(matrix[0, 2] + " vs " + matlabMatrix[254, 0]); //LoggedConsole.WriteLine(matrix[0, 3] + " vs " + matlabMatrix[253, 0]); // TODO put this back once sonogram issues resolved /* * LoggedConsole.WriteLine("START: AED"); * IEnumerable<Oblong> oblongs = AcousticEventDetection.detectEvents(3.0, 100, matrix); * LoggedConsole.WriteLine("END: AED"); * * * //set up static variables for init Acoustic events * //AcousticEvent. doMelScale = config.DoMelScale; * AcousticEvent.FreqBinCount = config.FreqBinCount; * AcousticEvent.FreqBinWidth = config.FftConfig.NyquistFreq / (double)config.FreqBinCount; * // int minF = (int)config.MinFreqBand; * // int maxF = (int)config.MaxFreqBand; * AcousticEvent.FrameDuration = config.GetFrameOffset(); * * * var events = new List<EventPatternRecog.Rectangle>(); * foreach (Oblong o in oblongs) * { * var e = new AcousticEvent(o); * events.Add(new EventPatternRecog.Rectangle(e.StartTime, (double) e.MaxFreq, e.StartTime + e.Duration, (double)e.MinFreq)); * //LoggedConsole.WriteLine(e.StartTime + "," + e.Duration + "," + e.MinFreq + "," + e.MaxFreq); * } * * LoggedConsole.WriteLine("# AED events: " + events.Count); * * LoggedConsole.WriteLine("START: EPR"); * IEnumerable<EventPatternRecog.Rectangle> eprRects = EventPatternRecog.detectGroundParrots(events); * LoggedConsole.WriteLine("END: EPR"); * * var eprEvents = new List<AcousticEvent>(); * foreach (EventPatternRecog.Rectangle r in eprRects) * { * var ae = new AcousticEvent(r.Left, r.Right - r.Left, r.Bottom, r.Top, false); * LoggedConsole.WriteLine(ae.WriteProperties()); * eprEvents.Add(ae); * } * * string imagePath = Path.Combine(outputFolder, "RESULTS_" + Path.GetFileNameWithoutExtension(recording.BaseName) + ".png"); * * bool doHighlightSubband = false; bool add1kHzLines = true; * var image = new Image_MultiTrack(sonogram.GetImage(doHighlightSubband, add1kHzLines)); * //image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration)); * //image.AddTrack(ImageTrack.GetWavEnvelopeTrack(recording, image.Image.Width)); * //image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); * image.AddEvents(eprEvents); * image.Save(outputFolder + wavFileName + ".png"); */ LoggedConsole.WriteLine("\nFINISHED!"); }
public void TestFeatureLearning() { // var outputDir = this.outputDirectory; var resultDir = PathHelper.ResolveAssetPath("FeatureLearning"); var folderPath = Path.Combine(resultDir, "random_audio_segments"); // Liz // PathHelper.ResolveAssetPath(@"C:\Users\kholghim\Mahnoosh\PcaWhitening\random_audio_segments\1192_1000"); // var resultDir = PathHelper.ResolveAssetPath(@"C:\Users\kholghim\Mahnoosh\PcaWhitening"); var outputMelImagePath = Path.Combine(resultDir, "MelScaleSpectrogram.png"); var outputNormMelImagePath = Path.Combine(resultDir, "NormalizedMelScaleSpectrogram.png"); var outputNoiseReducedMelImagePath = Path.Combine(resultDir, "NoiseReducedMelSpectrogram.png"); var outputReSpecImagePath = Path.Combine(resultDir, "ReconstrcutedSpectrogram.png"); // var outputClusterImagePath = Path.Combine(resultDir, "Clusters.bmp"); // +++++++++++++++++++++++++++++++++++++++++++++++++patch sampling from 1000 random 1-min recordings from Gympie // check whether there is any file in the folder/subfolders if (Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories).Length == 0) { throw new ArgumentException("The folder of recordings is empty..."); } // get the nyquist value from the first wav file in the folder of recordings int nq = new AudioRecording(Directory.GetFiles(folderPath, "*.wav")[0]).Nyquist; int nyquist = nq; // 11025; int frameSize = 1024; int finalBinCount = 128; // 256; // 100; // 40; // 200; // int hertzInterval = 1000; FreqScaleType scaleType = FreqScaleType.Mel; var freqScale = new FrequencyScale(scaleType, nyquist, frameSize, finalBinCount, hertzInterval); var fst = freqScale.ScaleType; var sonoConfig = new SonogramConfig { WindowSize = frameSize, // since each 24 frames duration is equal to 1 second WindowOverlap = 0.1028, DoMelScale = (scaleType == FreqScaleType.Mel) ? true : false, MelBinCount = (scaleType == FreqScaleType.Mel) ? finalBinCount : frameSize / 2, NoiseReductionType = NoiseReductionType.None, }; /* * // testing * var recordingPath3 = PathHelper.ResolveAsset(folderPath, "SM304264_0+1_20160421_024539_46-47min.wav"); * var recording3 = new AudioRecording(recordingPath3); * var sonogram3 = new SpectrogramStandard(sonoConfig, recording3.WavReader); * * // DO DRAW SPECTROGRAM * var image4 = sonogram3.GetImageFullyAnnotated(sonogram3.GetImage(), "MELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); * image4.Save(outputMelImagePath); * * // Do RMS normalization * sonogram3.Data = SNR.RmsNormalization(sonogram3.Data); * var image5 = sonogram3.GetImageFullyAnnotated(sonogram3.GetImage(), "NORMALISEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); * image5.Save(outputNormMelImagePath); * * // NOISE REDUCTION * sonogram3.Data = PcaWhitening.NoiseReduction(sonogram3.Data); * var image6 = sonogram3.GetImageFullyAnnotated(sonogram3.GetImage(), "NOISEREDUCEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); * image6.Save(outputNoiseReducedMelImagePath); * * //testing */ // Define the minFreBin and MaxFreqBin to be able to work at arbitrary frequency bin bounds. // The default value is minFreqBin = 1 and maxFreqBin = finalBinCount. // To work with arbitrary frequency bin bounds we need to manually set these two parameters. int minFreqBin = 40; //1 int maxFreqBin = 80; //finalBinCount; int numFreqBand = 1; //4; int patchWidth = (maxFreqBin - minFreqBin + 1) / numFreqBand; // finalBinCount / numFreqBand; int patchHeight = 1; // 2; // 4; // 16; // 6; // Frame size int numRandomPatches = 20; // 40; // 80; // 30; // 100; // 500; // // int fileCount = Directory.GetFiles(folderPath, "*.wav").Length; // Define variable number of "randomPatch" lists based on "numFreqBand" Dictionary <string, List <double[, ]> > randomPatchLists = new Dictionary <string, List <double[, ]> >(); for (int i = 0; i < numFreqBand; i++) { randomPatchLists.Add(string.Format("randomPatch{0}", i.ToString()), new List <double[, ]>()); } List <double[, ]> randomPatches = new List <double[, ]>(); /* * foreach (string filePath in Directory.GetFiles(folderPath, "*.wav")) * { * FileInfo f = filePath.ToFileInfo(); * if (f.Length == 0) * { * Debug.WriteLine(f.Name); * } * } */ double[,] inputMatrix; foreach (string filePath in Directory.GetFiles(folderPath, "*.wav")) { FileInfo fileInfo = filePath.ToFileInfo(); // process the wav file if it is not empty if (fileInfo.Length != 0) { var recording = new AudioRecording(filePath); sonoConfig.SourceFName = recording.BaseName; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // DO RMS NORMALIZATION sonogram.Data = SNR.RmsNormalization(sonogram.Data); // DO NOISE REDUCTION // sonogram.Data = SNR.NoiseReduce_Median(sonogram.Data, nhBackgroundThreshold: 2.0); sonogram.Data = PcaWhitening.NoiseReduction(sonogram.Data); // check whether the full band spectrogram is needed or a matrix with arbitrary freq bins if (minFreqBin != 1 || maxFreqBin != finalBinCount) { inputMatrix = PatchSampling.GetArbitraryFreqBandMatrix(sonogram.Data, minFreqBin, maxFreqBin); } else { inputMatrix = sonogram.Data; } // creating matrices from different freq bands of the source spectrogram List <double[, ]> allSubmatrices = PatchSampling.GetFreqBandMatrices(inputMatrix, numFreqBand); // Second: selecting random patches from each freq band matrix and add them to the corresponding patch list int count = 0; while (count < allSubmatrices.Count) { randomPatchLists[$"randomPatch{count.ToString()}"].Add(PatchSampling .GetPatches(allSubmatrices.ToArray()[count], patchWidth, patchHeight, numRandomPatches, PatchSampling.SamplingMethod.Random).ToMatrix()); count++; } } } foreach (string key in randomPatchLists.Keys) { randomPatches.Add(PatchSampling.ListOf2DArrayToOne2DArray(randomPatchLists[key])); } // convert list of random patches matrices to one matrix int numberOfClusters = 50; //256; // 128; // 64; // 32; // 10; // List <double[][]> allBandsCentroids = new List <double[][]>(); List <KMeansClusterCollection> allClusteringOutput = new List <KMeansClusterCollection>(); for (int i = 0; i < randomPatches.Count; i++) { double[,] patchMatrix = randomPatches[i]; // Apply PCA Whitening var whitenedSpectrogram = PcaWhitening.Whitening(true, patchMatrix); // Do k-means clustering var clusteringOutput = KmeansClustering.Clustering(whitenedSpectrogram.Reversion, numberOfClusters); // var clusteringOutput = KmeansClustering.Clustering(patchMatrix, noOfClusters, pathToClusterCsvFile); // writing centroids to a csv file // note that Csv.WriteToCsv can't write data types like dictionary<int, double[]> (problems with arrays) // I converted the dictionary values to a matrix and used the Csv.WriteMatrixToCsv // it might be a better way to do this string pathToClusterCsvFile = Path.Combine(resultDir, "ClusterCentroids" + i.ToString() + ".csv"); var clusterCentroids = clusteringOutput.ClusterIdCentroid.Values.ToArray(); Csv.WriteMatrixToCsv(pathToClusterCsvFile.ToFileInfo(), clusterCentroids.ToMatrix()); //Csv.WriteToCsv(pathToClusterCsvFile.ToFileInfo(), clusterCentroids); // sorting clusters based on size and output it to a csv file Dictionary <int, double> clusterIdSize = clusteringOutput.ClusterIdSize; int[] sortOrder = KmeansClustering.SortClustersBasedOnSize(clusterIdSize); // Write cluster ID and size to a CSV file string pathToClusterSizeCsvFile = Path.Combine(resultDir, "ClusterSize" + i.ToString() + ".csv"); Csv.WriteToCsv(pathToClusterSizeCsvFile.ToFileInfo(), clusterIdSize); // Draw cluster image directly from clustering output List <KeyValuePair <int, double[]> > list = clusteringOutput.ClusterIdCentroid.ToList(); double[][] centroids = new double[list.Count][]; for (int j = 0; j < list.Count; j++) { centroids[j] = list[j].Value; } allBandsCentroids.Add(centroids); allClusteringOutput.Add(clusteringOutput.Clusters); List <double[, ]> allCentroids = new List <double[, ]>(); for (int k = 0; k < centroids.Length; k++) { // convert each centroid to a matrix in order of cluster ID // double[,] cent = PatchSampling.ArrayToMatrixByColumn(centroids[i], patchWidth, patchHeight); // OR: in order of cluster size double[,] cent = MatrixTools.ArrayToMatrixByColumn(centroids[sortOrder[k]], patchWidth, patchHeight); // normalize each centroid double[,] normCent = DataTools.normalise(cent); // add a row of zero to each centroid double[,] cent2 = PatchSampling.AddRow(normCent); allCentroids.Add(cent2); } // concatenate all centroids double[,] mergedCentroidMatrix = PatchSampling.ListOf2DArrayToOne2DArray(allCentroids); // Draw clusters // int gridInterval = 1000; // var freqScale = new FrequencyScale(FreqScaleType.Mel, nyquist, frameSize, finalBinCount, gridInterval); var clusterImage = ImageTools.DrawMatrixWithoutNormalisation(mergedCentroidMatrix); clusterImage.RotateFlip(RotateFlipType.Rotate270FlipNone); // clusterImage.Save(outputClusterImagePath, ImageFormat.Bmp); var outputClusteringImage = Path.Combine(resultDir, "ClustersWithGrid" + i.ToString() + ".bmp"); // Image bmp = Image.Load<Rgb24>(filename); FrequencyScale.DrawFrequencyLinesOnImage((Image <Rgb24>)clusterImage, freqScale, includeLabels: false); clusterImage.Save(outputClusteringImage); } //+++++++++++++++++++++++++++++++++++++++++++++++++++++Processing and generating features for the target recordings var recording2Path = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav"); // var recording2Path = PathHelper.ResolveAsset(folderPath, "gympie_np_1192_353972_20160303_055854_60_0.wav"); // folder with 1000 files // var recording2Path = PathHelper.ResolveAsset(folderPath, "gympie_np_1192_353887_20151230_042625_60_0.wav"); // folder with 1000 files // var recording2Path = PathHelper.ResolveAsset(folderPath, "gympie_np_1192_354744_20151018_053923_60_0.wav"); // folder with 100 files var recording2 = new AudioRecording(recording2Path); var sonogram2 = new SpectrogramStandard(sonoConfig, recording2.WavReader); // DO DRAW SPECTROGRAM var image = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "MELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); image.Save(outputMelImagePath); // Do RMS normalization sonogram2.Data = SNR.RmsNormalization(sonogram2.Data); var image2 = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "NORMALISEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); image2.Save(outputNormMelImagePath); // NOISE REDUCTION sonogram2.Data = PcaWhitening.NoiseReduction(sonogram2.Data); var image3 = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "NOISEREDUCEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations); image3.Save(outputNoiseReducedMelImagePath); // check whether the full band spectrogram is needed or a matrix with arbitrary freq bins if (minFreqBin != 1 || maxFreqBin != finalBinCount) { inputMatrix = PatchSampling.GetArbitraryFreqBandMatrix(sonogram2.Data, minFreqBin, maxFreqBin); } else { inputMatrix = sonogram2.Data; } // extracting sequential patches from the target spectrogram List <double[, ]> allSubmatrices2 = PatchSampling.GetFreqBandMatrices(inputMatrix, numFreqBand); double[][,] matrices2 = allSubmatrices2.ToArray(); List <double[, ]> allSequentialPatchMatrix = new List <double[, ]>(); for (int i = 0; i < matrices2.GetLength(0); i++) { int rows = matrices2[i].GetLength(0); int columns = matrices2[i].GetLength(1); var sequentialPatches = PatchSampling.GetPatches(matrices2[i], patchWidth, patchHeight, (rows / patchHeight) * (columns / patchWidth), PatchSampling.SamplingMethod.Sequential); allSequentialPatchMatrix.Add(sequentialPatches.ToMatrix()); } // +++++++++++++++++++++++++++++++++++Feature Transformation // to do the feature transformation, we normalize centroids and // sequential patches from the input spectrogram to unit length // Then, we calculate the dot product of each patch with the centroids' matrix List <double[][]> allNormCentroids = new List <double[][]>(); for (int i = 0; i < allBandsCentroids.Count; i++) { // double check the index of the list double[][] normCentroids = new double[allBandsCentroids.ToArray()[i].GetLength(0)][]; for (int j = 0; j < allBandsCentroids.ToArray()[i].GetLength(0); j++) { normCentroids[j] = ART_2A.NormaliseVector(allBandsCentroids.ToArray()[i][j]); } allNormCentroids.Add(normCentroids); } List <double[][]> allFeatureTransVectors = new List <double[][]>(); for (int i = 0; i < allSequentialPatchMatrix.Count; i++) { double[][] featureTransVectors = new double[allSequentialPatchMatrix.ToArray()[i].GetLength(0)][]; for (int j = 0; j < allSequentialPatchMatrix.ToArray()[i].GetLength(0); j++) { var normVector = ART_2A.NormaliseVector(allSequentialPatchMatrix.ToArray()[i] .ToJagged()[j]); // normalize each patch to unit length featureTransVectors[j] = allNormCentroids.ToArray()[i].ToMatrix().Dot(normVector); } allFeatureTransVectors.Add(featureTransVectors); } // +++++++++++++++++++++++++++++++++++Feature Transformation // +++++++++++++++++++++++++++++++++++Temporal Summarization // The resolution to generate features is 1 second // Each 24 single-frame patches form 1 second // for each 24 patch, we generate 3 vectors of mean, std, and max // The pre-assumption is that each input spectrogram is 1 minute List <double[, ]> allMeanFeatureVectors = new List <double[, ]>(); List <double[, ]> allMaxFeatureVectors = new List <double[, ]>(); List <double[, ]> allStdFeatureVectors = new List <double[, ]>(); // number of frames needs to be concatenated to form 1 second. Each 24 frames make 1 second. int numFrames = (24 / patchHeight) * 60; foreach (var freqBandFeature in allFeatureTransVectors) { // store features of different bands in lists List <double[]> meanFeatureVectors = new List <double[]>(); List <double[]> maxFeatureVectors = new List <double[]>(); List <double[]> stdFeatureVectors = new List <double[]>(); int c = 0; while (c + numFrames < freqBandFeature.GetLength(0)) { // First, make a list of patches that would be equal to 1 second List <double[]> sequencesOfFramesList = new List <double[]>(); for (int i = c; i < c + numFrames; i++) { sequencesOfFramesList.Add(freqBandFeature[i]); } List <double> mean = new List <double>(); List <double> std = new List <double>(); List <double> max = new List <double>(); double[,] sequencesOfFrames = sequencesOfFramesList.ToArray().ToMatrix(); // int len = sequencesOfFrames.GetLength(1); // Second, calculate mean, max, and standard deviation of six vectors element-wise for (int j = 0; j < sequencesOfFrames.GetLength(1); j++) { double[] temp = new double[sequencesOfFrames.GetLength(0)]; for (int k = 0; k < sequencesOfFrames.GetLength(0); k++) { temp[k] = sequencesOfFrames[k, j]; } mean.Add(AutoAndCrossCorrelation.GetAverage(temp)); std.Add(AutoAndCrossCorrelation.GetStdev(temp)); max.Add(temp.GetMaxValue()); } meanFeatureVectors.Add(mean.ToArray()); maxFeatureVectors.Add(max.ToArray()); stdFeatureVectors.Add(std.ToArray()); c += numFrames; } allMeanFeatureVectors.Add(meanFeatureVectors.ToArray().ToMatrix()); allMaxFeatureVectors.Add(maxFeatureVectors.ToArray().ToMatrix()); allStdFeatureVectors.Add(stdFeatureVectors.ToArray().ToMatrix()); } // +++++++++++++++++++++++++++++++++++Temporal Summarization // ++++++++++++++++++++++++++++++++++Writing features to file // First, concatenate mean, max, std for each second. // Then write to CSV file. for (int j = 0; j < allMeanFeatureVectors.Count; j++) { // write the features of each pre-defined frequency band into a separate CSV file var outputFeatureFile = Path.Combine(resultDir, "FeatureVectors" + j.ToString() + ".csv"); // creating the header for CSV file List <string> header = new List <string>(); for (int i = 0; i < allMeanFeatureVectors.ToArray()[j].GetLength(1); i++) { header.Add("mean" + i.ToString()); } for (int i = 0; i < allMaxFeatureVectors.ToArray()[j].GetLength(1); i++) { header.Add("max" + i.ToString()); } for (int i = 0; i < allStdFeatureVectors.ToArray()[j].GetLength(1); i++) { header.Add("std" + i.ToString()); } // concatenating mean, std, and max vector together for each 1 second List <double[]> featureVectors = new List <double[]>(); for (int i = 0; i < allMeanFeatureVectors.ToArray()[j].ToJagged().GetLength(0); i++) { List <double[]> featureList = new List <double[]> { allMeanFeatureVectors.ToArray()[j].ToJagged()[i], allMaxFeatureVectors.ToArray()[j].ToJagged()[i], allStdFeatureVectors.ToArray()[j].ToJagged()[i], }; double[] featureVector = DataTools.ConcatenateVectors(featureList); featureVectors.Add(featureVector); } // writing feature vectors to CSV file using (StreamWriter file = new StreamWriter(outputFeatureFile)) { // writing the header to CSV file foreach (var entry in header.ToArray()) { file.Write(entry + ","); } file.Write(Environment.NewLine); foreach (var entry in featureVectors.ToArray()) { foreach (var value in entry) { file.Write(value + ","); } file.Write(Environment.NewLine); } } } /* * // Reconstructing the target spectrogram based on clusters' centroids * List<double[,]> convertedSpec = new List<double[,]>(); * int columnPerFreqBand = sonogram2.Data.GetLength(1) / numFreqBand; * for (int i = 0; i < allSequentialPatchMatrix.Count; i++) * { * double[,] reconstructedSpec2 = KmeansClustering.ReconstructSpectrogram(allSequentialPatchMatrix.ToArray()[i], allClusteringOutput.ToArray()[i]); * convertedSpec.Add(PatchSampling.ConvertPatches(reconstructedSpec2, patchWidth, patchHeight, columnPerFreqBand)); * } * * sonogram2.Data = PatchSampling.ConcatFreqBandMatrices(convertedSpec); * * // DO DRAW SPECTROGRAM * var reconstructedSpecImage = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "RECONSTRUCTEDSPECTROGRAM: " + freqScale.ScaleType.ToString(), freqScale.GridLineLocations); * reconstructedSpecImage.Save(outputReSpecImagePath); */ }
public void TestKmeansClustering() { var outputDir = this.outputDirectory; var recordingsPath = PathHelper.ResolveAssetPath("FeatureLearning"); var folderPath = Path.Combine(recordingsPath, "random_audio_segments"); var outputImagePath = Path.Combine(outputDir.FullName, "ReconstrcutedSpectrogram.png"); // check whether there is any file in the folder/subfolders if (Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories).Length == 0) { throw new ArgumentException("The folder of recordings is empty. Test will fail!"); } // get the nyquist value from the first wav file in the folder of recordings int nq = new AudioRecording(Directory.GetFiles(folderPath, "*.wav")[0]).Nyquist; int nyquist = nq; int frameSize = 1024; int finalBinCount = 128; int hertzInterval = 1000; FreqScaleType scaleType = FreqScaleType.Mel; var freqScale = new FrequencyScale(scaleType, nyquist, frameSize, finalBinCount, hertzInterval); var sonoConfig = new SonogramConfig { WindowSize = frameSize, //WindowOverlap is set based on the fact that each 24 frames is equal to 1 second WindowOverlap = 0.1028, DoMelScale = (scaleType == FreqScaleType.Mel) ? true : false, MelBinCount = (scaleType == FreqScaleType.Mel) ? finalBinCount : frameSize / 2, NoiseReductionType = NoiseReductionType.None, }; int numberOfFreqBand = 4; int patchWidth = finalBinCount / numberOfFreqBand; int patchHeight = 1; int numberOfRandomPatches = 20; // Define variable number of "randomPatch" lists based on "numberOfFreqBand" Dictionary <string, List <double[, ]> > randomPatchLists = new Dictionary <string, List <double[, ]> >(); for (int i = 0; i < numberOfFreqBand; i++) { randomPatchLists.Add(string.Format("randomPatch{0}", i.ToString()), new List <double[, ]>()); } List <double[, ]> randomPatches = new List <double[, ]>(); foreach (string filePath in Directory.GetFiles(folderPath, "*.wav")) { FileInfo fileInfo = filePath.ToFileInfo(); // process the wav file if it is not empty if (fileInfo.Length != 0) { var recording = new AudioRecording(filePath); sonoConfig.SourceFName = recording.BaseName; var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader); // DO RMS NORMALIZATION sonogram.Data = SNR.RmsNormalization(sonogram.Data); // DO NOISE REDUCTION sonogram.Data = PcaWhitening.NoiseReduction(sonogram.Data); // creating matrices from different freq bands of the source spectrogram List <double[, ]> allSubmatrices = PatchSampling.GetFreqBandMatrices(sonogram.Data, numberOfFreqBand); // Second: selecting random patches from each freq band matrix and add them to the corresponding patch list int count = 0; while (count < allSubmatrices.Count) { randomPatchLists[string.Format("randomPatch{0}", count.ToString())].Add(PatchSampling.GetPatches(allSubmatrices.ToArray()[count], patchWidth, patchHeight, numberOfRandomPatches, PatchSampling.SamplingMethod.Random).ToMatrix()); count++; } } } foreach (string key in randomPatchLists.Keys) { randomPatches.Add(PatchSampling.ListOf2DArrayToOne2DArray(randomPatchLists[key])); } // convert list of random patches matrices to one matrix int numberOfClusters = 32; List <KMeansClusterCollection> allClusteringOutput = new List <KMeansClusterCollection>(); for (int i = 0; i < randomPatches.Count; i++) { double[,] patchMatrix = randomPatches[i]; // Do k-means clustering string pathToClusterCsvFile = Path.Combine(outputDir.FullName, "ClusterCentroids" + i.ToString() + ".csv"); var clusteringOutput = KmeansClustering.Clustering(patchMatrix, numberOfClusters); // sorting clusters based on size and output it to a csv file Dictionary <int, double> clusterIdSize = clusteringOutput.ClusterIdSize; int[] sortOrder = KmeansClustering.SortClustersBasedOnSize(clusterIdSize); // Write cluster ID and size to a CSV file string pathToClusterSizeCsvFile = Path.Combine(outputDir.FullName, "ClusterSize" + i.ToString() + ".csv"); Csv.WriteToCsv(pathToClusterSizeCsvFile.ToFileInfo(), clusterIdSize); // Draw cluster image directly from clustering output List <KeyValuePair <int, double[]> > listCluster = clusteringOutput.ClusterIdCentroid.ToList(); double[][] centroids = new double[listCluster.Count][]; for (int j = 0; j < listCluster.Count; j++) { centroids[j] = listCluster[j].Value; } allClusteringOutput.Add(clusteringOutput.Clusters); List <double[, ]> allCentroids = new List <double[, ]>(); for (int k = 0; k < centroids.Length; k++) { // convert each centroid to a matrix in order of cluster ID // OR: in order of cluster size double[,] centroid = MatrixTools.ArrayToMatrixByColumn(centroids[sortOrder[k]], patchWidth, patchHeight); // normalize each centroid double[,] normalizedCentroid = DataTools.normalise(centroid); // add a row of zero to each centroid double[,] newCentroid = PatchSampling.AddRow(normalizedCentroid); allCentroids.Add(newCentroid); } // concatenate all centroids double[,] mergedCentroidMatrix = PatchSampling.ListOf2DArrayToOne2DArray(allCentroids); // Draw clusters var clusterImage = ImageTools.DrawMatrixWithoutNormalisation(mergedCentroidMatrix); clusterImage.RotateFlip(RotateFlipType.Rotate270FlipNone); var outputClusteringImage = Path.Combine(outputDir.FullName, "ClustersWithGrid" + i.ToString() + ".bmp"); FrequencyScale.DrawFrequencyLinesOnImage((Bitmap)clusterImage, freqScale, includeLabels: false); clusterImage.Save(outputClusteringImage); } //+++++++++++++++++++++++++++++++++++++++++++Reconstructing a target spectrogram from sequential patches and the cluster centroids var recording2Path = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav"); var recording2 = new AudioRecording(recording2Path); var sonogram2 = new SpectrogramStandard(sonoConfig, recording2.WavReader); var targetSpec = sonogram2.Data; // Do RMS normalization sonogram2.Data = SNR.RmsNormalization(sonogram2.Data); // NOISE REDUCTION sonogram2.Data = PcaWhitening.NoiseReduction(sonogram2.Data); // extracting sequential patches from the target spectrogram List <double[, ]> allSubmatrices2 = PatchSampling.GetFreqBandMatrices(sonogram2.Data, numberOfFreqBand); double[][,] matrices2 = allSubmatrices2.ToArray(); List <double[, ]> allSequentialPatchMatrix = new List <double[, ]>(); for (int i = 0; i < matrices2.GetLength(0); i++) { int rows = matrices2[i].GetLength(0); int columns = matrices2[i].GetLength(1); var sequentialPatches = PatchSampling.GetPatches(matrices2[i], patchWidth, patchHeight, (rows / patchHeight) * (columns / patchWidth), PatchSampling.SamplingMethod.Sequential); allSequentialPatchMatrix.Add(sequentialPatches.ToMatrix()); } List <double[, ]> convertedSpectrogram = new List <double[, ]>(); int columnPerFreqBand = sonogram2.Data.GetLength(1) / numberOfFreqBand; for (int i = 0; i < allSequentialPatchMatrix.Count; i++) { double[,] reconstructedSpec2 = KmeansClustering.ReconstructSpectrogram(allSequentialPatchMatrix.ToArray()[i], allClusteringOutput.ToArray()[i]); convertedSpectrogram.Add(PatchSampling.ConvertPatches(reconstructedSpec2, patchWidth, patchHeight, columnPerFreqBand)); } sonogram2.Data = PatchSampling.ConcatFreqBandMatrices(convertedSpectrogram); // DO DRAW SPECTROGRAM var reconstructedSpecImage = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "RECONSTRUCTEDSPECTROGRAM: " + freqScale.ScaleType.ToString(), freqScale.GridLineLocations); reconstructedSpecImage.Save(outputImagePath, ImageFormat.Png); // DO UNIT TESTING Assert.AreEqual(targetSpec.GetLength(0), sonogram2.Data.GetLength(0)); Assert.AreEqual(targetSpec.GetLength(1), sonogram2.Data.GetLength(1)); }