/// <summary> /// This CLUSTERING method is called only from IndexCalculate.cs /// and TESTMETHOD_SpectralClustering(string wavFilePath, string outputDir, int frameSize) /// It estimates the number of spectral clusters in a spectrogram, /// and outputs two summary indices: cluster count (also called spectral diversity) and the threegram count. /// IMPORTANT NOTE: The passed spectrogram MUST be already noise reduced. /// This clustering algorithm is a highly reduced version of binary ART, Adaptive resonance Theory, designed for speed. /// </summary> /// <param name="spectrogram">a collection of spectra that are to be clustered</param> /// <param name="lowerBinBound">lower end of the bird-band</param> /// <param name="upperBinBound">upper end of the bird-band</param> /// <param name="binaryThreshold">used to convert real value spectrum to binary</param> public static ClusterInfo ClusterTheSpectra(double[,] spectrogram, int lowerBinBound, int upperBinBound, double binaryThreshold) { // Use verbose only when debugging // SpectralClustering.Verbose = true; var parameters = new ClusteringParameters(lowerBinBound, upperBinBound, binaryThreshold, DefaultRowSumThreshold); TrainingDataInfo data = GetTrainingDataForClustering(spectrogram, parameters); // cluster pruning parameters const double weightThreshold = DefaultRowSumThreshold; // used to remove weight vectors whose sum of wts <= threshold const int hitThreshold = DefaultHitThreshold; // used to remove weight vectors which have fewer than the threshold number of hits i.e. sets minimum cluster size // Skip clustering if not enough suitable training data if (data.TrainingData.Count <= 8) { int freqBinCount = spectrogram.GetLength(1); return(new ClusterInfo(freqBinCount)); } var clusterInfo = ClusterAnalysis(data.TrainingData, weightThreshold, hitThreshold, data.SelectedFrames); return(clusterInfo); }
/// <summary> /// Overlays the spectral cluster IDs on a spectrogram from which the clusters derived. /// </summary> public static Image DrawClusterSpectrogram(BaseSonogram sonogram, ClusterInfo clusterInfo, TrainingDataInfo data, int lowerBinBound) { using (var img = sonogram.GetImage(doHighlightSubband: false, add1KHzLines: true, doMelScale: false)) using (var image = new Image_MultiTrack(img)) { //image.AddTrack(ImageTrack.GetScoreTrack(DataTools.Bool2Binary(clusterInfo.selectedFrames),0.0, 1.0, 0.0)); //add time scale image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond)); image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); // add cluster track, show ID of cluster of each frame string label = string.Format(clusterInfo.ClusterCount + " Clusters"); var scores = new Plot(label, DataTools.normalise(clusterInfo.ClusterHits2), 0.0); // location of cluster hits image.AddTrack(ImageTrack.GetNamedScoreTrack(scores.data, 0.0, 1.0, scores.threshold, scores.title)); // overlay cluster hits on spectrogram int[,] hits = AssembleClusterSpectrogram(sonogram.Data, lowerBinBound, clusterInfo, data); image.OverlayDiscreteColorMatrix(hits); return(image.GetImage()); }// using }
/// <summary> /// This method was set up as a TESTMETHOD in May 2017 but has not yet been debugged. /// It was transferred from Sandpit.cls. It is several years old. /// Updated May 2017. /// </summary> public static void TESTMETHOD_SpectralClustering(string wavFilePath, string outputDir, int frameSize) { string imageViewer = @"C:\Windows\system32\mspaint.exe"; var recording = new AudioRecording(wavFilePath); // get recording segment // test clustering using amplitude spectrogram // double binaryThreshold = 0.07; // An amplitude threshold of 0.03 = -30 dB. A threshold of 0.05 = -26dB. // double[,] spectrogramData = GetAmplitudeSpectrogramNoiseReduced(recording, frameSize); // test clustering using decibel spectrogram double binaryThreshold = DefaultBinaryThresholdInDecibels; // A decibel threshold for converting to binary double[,] spectrogramData = GetDecibelSpectrogramNoiseReduced(recording, frameSize); //####################################################################################################################################### int nyquistFreq = recording.Nyquist; int frameCount = spectrogramData.GetLength(0); int freqBinCount = spectrogramData.GetLength(1); double binWidth = nyquistFreq / (double)freqBinCount; // We only use the midband of the Spectrogram, i.e. the band between lowerBinBound and upperBinBound. // int lowFreqBound = 1000; // int upperBinBound = freqBinCount - 5; // In June 2016, the mid-band was set to lowerBound=1000Hz, upperBound=8000hz, because this band contains most bird activity, i.e. it is the Bird-Band // This was done in order to make the cluster summary indices more reflective of bird call activity. // int lowerFreqBound = 482; int lowerFreqBound = 1000; int lowerBinBound = (int)Math.Ceiling(lowerFreqBound / binWidth); int upperFreqBound = 8000; int upperBinBound = (int)Math.Ceiling(upperFreqBound / binWidth); var midBandSpectrogram = MatrixTools.Submatrix(spectrogramData, 0, lowerBinBound, spectrogramData.GetLength(0) - 1, upperBinBound); var clusterInfo = ClusterTheSpectra(midBandSpectrogram, lowerBinBound, upperBinBound, binaryThreshold); // transfer cluster info to spectral index results var clusterSpectrum1 = RestoreFullLengthSpectrum(clusterInfo.ClusterSpectrum, freqBinCount, lowerBinBound); Console.WriteLine("Lower BinBound=" + lowerBinBound); Console.WriteLine("Upper BinBound=" + upperBinBound); Console.WriteLine("Binary Threshold=" + binaryThreshold); Console.WriteLine("Cluster Count =" + clusterInfo.ClusterCount); Console.WriteLine("Three Gram Count=" + clusterInfo.TriGramUniqueCount); // ################################################################################### // Now repeat the entire process again. This time we want to get intermediate results to superimpose on spectrogram // Need to specify additional parameters. ACTIVITY THRESHOLD - require activity in at least N bins to include spectrum for training double rowSumThreshold = DefaultRowSumThreshold; var parameters = new ClusteringParameters(lowerBinBound, upperBinBound, binaryThreshold, rowSumThreshold); TrainingDataInfo data = GetTrainingDataForClustering(midBandSpectrogram, parameters); // make a normal standard decibel spectogram on which to superimpose cluster results var stdSpectrogram = GetStandardSpectrogram(recording, frameSize); var image = DrawSonogram(stdSpectrogram, null, null, 0.0, null); SaveAndViewSpectrogramImage(image, outputDir, "test0Spectrogram.png", imageViewer); // Debug.Assert(data.TrainingDataAsSpectrogram != null, "data.TrainingDataAsSpectrogram != null"); double[,] overlay = ConvertOverlayToSpectrogramSize(data.TrainingDataAsSpectrogram, lowerBinBound, frameCount, freqBinCount); image = DrawSonogram(stdSpectrogram, null, null, 0.0, overlay); SaveAndViewSpectrogramImage(image, outputDir, "test1Spectrogram.png", imageViewer); // Return if no suitable training data for clustering if (data.TrainingData.Count <= 8) { Console.WriteLine("Abort clustering. Only {0} spectra available for training data. Must be at least 9.", data.TrainingData.Count); } else { // the following are pruning parameters double wtThreshold = rowSumThreshold; // used to remove wt vectors whose sum of wts <= threshold int hitThreshold = DefaultHitThreshold; // used to remove wt vectors which have < N hits, i.e. cluster size < N clusterInfo = ClusterAnalysis(data.TrainingData, wtThreshold, hitThreshold, data.SelectedFrames); Console.WriteLine("Binary Threshold=" + binaryThreshold); Console.WriteLine("Weight Threshold=" + wtThreshold); Console.WriteLine("Hit Threshold=" + hitThreshold); Console.WriteLine("Cluster Count=" + clusterInfo.ClusterCount); Console.WriteLine("Three Gram Count=" + clusterInfo.TriGramUniqueCount); image = DrawClusterSpectrogram(stdSpectrogram, clusterInfo, data, lowerBinBound); SaveAndViewSpectrogramImage(image, outputDir, "test2Spectrogram.png", imageViewer); // transfer cluster info to spectral index results var clusterSpectrum2 = RestoreFullLengthSpectrum(clusterInfo.ClusterSpectrum, freqBinCount, lowerBinBound); TestTools.CompareTwoArrays(clusterSpectrum1, clusterSpectrum2); } }
/// <summary> /// this method is used only to visualize the clusters and which frames they hit. /// Create a new spectrogram of same size as the passed spectrogram. /// Later on it is superimposed on a detailed spectrogram. /// </summary> /// <param name="spectrogram">spectrogram used to derive spectral richness indices. Orientation is row=frame</param> /// <param name="lowerBinBound">bottom N freq bins are excluded because likely to contain traffic and wind noise.</param> /// <param name="clusterInfo">information about accumulated clusters</param> /// <param name="data">training data</param> public static int[,] AssembleClusterSpectrogram(double[,] spectrogram, int lowerBinBound, ClusterInfo clusterInfo, TrainingDataInfo data) { // the weight vector for each cluster - a list of double-arrays var clusterWts = clusterInfo.PrunedClusterWts; // an array indicating which cluster each frame belongs to. Zero = no cluster int[] clusterHits = clusterInfo.ClusterHits2; bool[] activeFrames = clusterInfo.SelectedFrames; int frameCount = spectrogram.GetLength(0); int freqBinCount = spectrogram.GetLength(1); //reassemble spectrogram to visualise the clusters var clusterSpectrogram = new int[frameCount, freqBinCount]; for (int row = 0; row < frameCount; row++) { if (activeFrames[row]) { int clusterId = clusterHits[row]; if (clusterId == 0) { // cluster zero does not exist. Place filler continue; } double[] wtVector = clusterWts[clusterId]; if (wtVector == null) { // This should not happen but ... LoggedConsole.WriteErrorLine($"WARNING: Cluster {clusterId} = null"); continue; } double[] fullLengthSpectrum = RestoreFullLengthSpectrum(wtVector, freqBinCount, data.LowBinBound); //for (int j = lowerBinBound; j < freqBinCount; j++) for (int j = 0; j < freqBinCount; j++) { //if (spectrogram[row, j] > data.intensityThreshold) if (fullLengthSpectrum[j] > 0.0) { clusterSpectrogram[row, j] = clusterId + 1; //+1 so do not have zero index for a cluster } else { clusterSpectrogram[row, j] = 0; //correct for case where set hit count < 0 for pruned wts. } } } } //add in the weights to first part of spectrogram //int space = 10; //int col = space; //for (int i = 0; i < clusterWts.Count; i++) //{ // if (clusterWts[i] == null) continue; // for (int c = 0; c < space; c++) // { // col++; // //for (int j = 0; j < clusterSpectrogram.GetLength(1); j++) clusterSpectrogram[col, j] = clusterWts.Count+3; // for (int j = 0; j < clusterWts[i].Length; j++) // { // if (clusterWts[i][j] > 0.0) clusterSpectrogram[col, excludeBins + j - 1] = i + 1; // } // } // //col += 2; //} return(clusterSpectrogram); }