/// <summary>
/// This CLUSTERING method is called only from IndexCalculate.cs
/// and TESTMETHOD_SpectralClustering(string wavFilePath, string outputDir, int frameSize)
/// It estimates the number of spectral clusters in a spectrogram,
/// and outputs two summary indices: cluster count (also called spectral diversity) and the threegram count.
/// IMPORTANT NOTE: The passed spectrogram MUST be already noise reduced.
/// This clustering algorithm is a highly reduced version of binary ART, Adaptive resonance Theory, designed for speed.
/// </summary>
/// <param name="spectrogram">a collection of spectra that are to be clustered</param>
/// <param name="lowerBinBound">lower end of the bird-band</param>
/// <param name="upperBinBound">upper end of the bird-band</param>
/// <param name="binaryThreshold">used to convert real value spectrum to binary</param>
public static ClusterInfo ClusterTheSpectra(double[,] spectrogram, int lowerBinBound, int upperBinBound, double binaryThreshold)
{
// Use verbose only when debugging
// SpectralClustering.Verbose = true;
var parameters = new ClusteringParameters(lowerBinBound, upperBinBound, binaryThreshold, DefaultRowSumThreshold);
TrainingDataInfo data = GetTrainingDataForClustering(spectrogram, parameters);
// cluster pruning parameters
const double weightThreshold = DefaultRowSumThreshold; // used to remove weight vectors whose sum of wts <= threshold
const int hitThreshold = DefaultHitThreshold; // used to remove weight vectors which have fewer than the threshold number of hits i.e. sets minimum cluster size
// Skip clustering if not enough suitable training data
if (data.TrainingData.Count <= 8)
{
int freqBinCount = spectrogram.GetLength(1);
return(new ClusterInfo(freqBinCount));
}
var clusterInfo = ClusterAnalysis(data.TrainingData, weightThreshold, hitThreshold, data.SelectedFrames);
return(clusterInfo);
}
/// <summary>
/// Overlays the spectral cluster IDs on a spectrogram from which the clusters derived.
/// </summary>
public static Image DrawClusterSpectrogram(BaseSonogram sonogram, ClusterInfo clusterInfo, TrainingDataInfo data, int lowerBinBound)
{
using (var img = sonogram.GetImage(doHighlightSubband: false, add1KHzLines: true, doMelScale: false))
using (var image = new Image_MultiTrack(img))
{
//image.AddTrack(ImageTrack.GetScoreTrack(DataTools.Bool2Binary(clusterInfo.selectedFrames),0.0, 1.0, 0.0));
//add time scale
image.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond));
image.AddTrack(ImageTrack.GetSegmentationTrack(sonogram));
// add cluster track, show ID of cluster of each frame
string label = string.Format(clusterInfo.ClusterCount + " Clusters");
var scores = new Plot(label, DataTools.normalise(clusterInfo.ClusterHits2), 0.0); // location of cluster hits
image.AddTrack(ImageTrack.GetNamedScoreTrack(scores.data, 0.0, 1.0, scores.threshold, scores.title));
// overlay cluster hits on spectrogram
int[,] hits = AssembleClusterSpectrogram(sonogram.Data, lowerBinBound, clusterInfo, data);
image.OverlayDiscreteColorMatrix(hits);
return(image.GetImage());
}// using
}
/// <summary>
/// This method was set up as a TESTMETHOD in May 2017 but has not yet been debugged.
/// It was transferred from Sandpit.cls. It is several years old.
/// Updated May 2017.
/// </summary>
public static void TESTMETHOD_SpectralClustering(string wavFilePath, string outputDir, int frameSize)
{
string imageViewer = @"C:\Windows\system32\mspaint.exe";
var recording = new AudioRecording(wavFilePath); // get recording segment
// test clustering using amplitude spectrogram
// double binaryThreshold = 0.07; // An amplitude threshold of 0.03 = -30 dB. A threshold of 0.05 = -26dB.
// double[,] spectrogramData = GetAmplitudeSpectrogramNoiseReduced(recording, frameSize);
// test clustering using decibel spectrogram
double binaryThreshold = DefaultBinaryThresholdInDecibels; // A decibel threshold for converting to binary
double[,] spectrogramData = GetDecibelSpectrogramNoiseReduced(recording, frameSize);
//#######################################################################################################################################
int nyquistFreq = recording.Nyquist;
int frameCount = spectrogramData.GetLength(0);
int freqBinCount = spectrogramData.GetLength(1);
double binWidth = nyquistFreq / (double)freqBinCount;
// We only use the midband of the Spectrogram, i.e. the band between lowerBinBound and upperBinBound.
// int lowFreqBound = 1000;
// int upperBinBound = freqBinCount - 5;
// In June 2016, the mid-band was set to lowerBound=1000Hz, upperBound=8000hz, because this band contains most bird activity, i.e. it is the Bird-Band
// This was done in order to make the cluster summary indices more reflective of bird call activity.
// int lowerFreqBound = 482;
int lowerFreqBound = 1000;
int lowerBinBound = (int)Math.Ceiling(lowerFreqBound / binWidth);
int upperFreqBound = 8000;
int upperBinBound = (int)Math.Ceiling(upperFreqBound / binWidth);
var midBandSpectrogram = MatrixTools.Submatrix(spectrogramData, 0, lowerBinBound, spectrogramData.GetLength(0) - 1, upperBinBound);
var clusterInfo = ClusterTheSpectra(midBandSpectrogram, lowerBinBound, upperBinBound, binaryThreshold);
// transfer cluster info to spectral index results
var clusterSpectrum1 = RestoreFullLengthSpectrum(clusterInfo.ClusterSpectrum, freqBinCount, lowerBinBound);
Console.WriteLine("Lower BinBound=" + lowerBinBound);
Console.WriteLine("Upper BinBound=" + upperBinBound);
Console.WriteLine("Binary Threshold=" + binaryThreshold);
Console.WriteLine("Cluster Count =" + clusterInfo.ClusterCount);
Console.WriteLine("Three Gram Count=" + clusterInfo.TriGramUniqueCount);
// ###################################################################################
// Now repeat the entire process again. This time we want to get intermediate results to superimpose on spectrogram
// Need to specify additional parameters. ACTIVITY THRESHOLD - require activity in at least N bins to include spectrum for training
double rowSumThreshold = DefaultRowSumThreshold;
var parameters = new ClusteringParameters(lowerBinBound, upperBinBound, binaryThreshold, rowSumThreshold);
TrainingDataInfo data = GetTrainingDataForClustering(midBandSpectrogram, parameters);
// make a normal standard decibel spectogram on which to superimpose cluster results
var stdSpectrogram = GetStandardSpectrogram(recording, frameSize);
var image = DrawSonogram(stdSpectrogram, null, null, 0.0, null);
SaveAndViewSpectrogramImage(image, outputDir, "test0Spectrogram.png", imageViewer);
// Debug.Assert(data.TrainingDataAsSpectrogram != null, "data.TrainingDataAsSpectrogram != null");
double[,] overlay = ConvertOverlayToSpectrogramSize(data.TrainingDataAsSpectrogram, lowerBinBound, frameCount, freqBinCount);
image = DrawSonogram(stdSpectrogram, null, null, 0.0, overlay);
SaveAndViewSpectrogramImage(image, outputDir, "test1Spectrogram.png", imageViewer);
// Return if no suitable training data for clustering
if (data.TrainingData.Count <= 8)
{
Console.WriteLine("Abort clustering. Only {0} spectra available for training data. Must be at least 9.", data.TrainingData.Count);
}
else
{
// the following are pruning parameters
double wtThreshold = rowSumThreshold; // used to remove wt vectors whose sum of wts <= threshold
int hitThreshold = DefaultHitThreshold; // used to remove wt vectors which have < N hits, i.e. cluster size < N
clusterInfo = ClusterAnalysis(data.TrainingData, wtThreshold, hitThreshold, data.SelectedFrames);
Console.WriteLine("Binary Threshold=" + binaryThreshold);
Console.WriteLine("Weight Threshold=" + wtThreshold);
Console.WriteLine("Hit Threshold=" + hitThreshold);
Console.WriteLine("Cluster Count=" + clusterInfo.ClusterCount);
Console.WriteLine("Three Gram Count=" + clusterInfo.TriGramUniqueCount);
image = DrawClusterSpectrogram(stdSpectrogram, clusterInfo, data, lowerBinBound);
SaveAndViewSpectrogramImage(image, outputDir, "test2Spectrogram.png", imageViewer);
// transfer cluster info to spectral index results
var clusterSpectrum2 = RestoreFullLengthSpectrum(clusterInfo.ClusterSpectrum, freqBinCount, lowerBinBound);
TestTools.CompareTwoArrays(clusterSpectrum1, clusterSpectrum2);
}
}
/// <summary>
/// this method is used only to visualize the clusters and which frames they hit.
/// Create a new spectrogram of same size as the passed spectrogram.
/// Later on it is superimposed on a detailed spectrogram.
/// </summary>
/// <param name="spectrogram">spectrogram used to derive spectral richness indices. Orientation is row=frame</param>
/// <param name="lowerBinBound">bottom N freq bins are excluded because likely to contain traffic and wind noise.</param>
/// <param name="clusterInfo">information about accumulated clusters</param>
/// <param name="data">training data</param>
public static int[,] AssembleClusterSpectrogram(double[,] spectrogram, int lowerBinBound, ClusterInfo clusterInfo, TrainingDataInfo data)
{
// the weight vector for each cluster - a list of double-arrays
var clusterWts = clusterInfo.PrunedClusterWts;
// an array indicating which cluster each frame belongs to. Zero = no cluster
int[] clusterHits = clusterInfo.ClusterHits2;
bool[] activeFrames = clusterInfo.SelectedFrames;
int frameCount = spectrogram.GetLength(0);
int freqBinCount = spectrogram.GetLength(1);
//reassemble spectrogram to visualise the clusters
var clusterSpectrogram = new int[frameCount, freqBinCount];
for (int row = 0; row < frameCount; row++)
{
if (activeFrames[row])
{
int clusterId = clusterHits[row];
if (clusterId == 0)
{
// cluster zero does not exist. Place filler
continue;
}
double[] wtVector = clusterWts[clusterId];
if (wtVector == null)
{
// This should not happen but ...
LoggedConsole.WriteErrorLine($"WARNING: Cluster {clusterId} = null");
continue;
}
double[] fullLengthSpectrum = RestoreFullLengthSpectrum(wtVector, freqBinCount, data.LowBinBound);
//for (int j = lowerBinBound; j < freqBinCount; j++)
for (int j = 0; j < freqBinCount; j++)
{
//if (spectrogram[row, j] > data.intensityThreshold)
if (fullLengthSpectrum[j] > 0.0)
{
clusterSpectrogram[row, j] = clusterId + 1; //+1 so do not have zero index for a cluster
}
else
{
clusterSpectrogram[row, j] = 0; //correct for case where set hit count < 0 for pruned wts.
}
}
}
}
//add in the weights to first part of spectrogram
//int space = 10;
//int col = space;
//for (int i = 0; i < clusterWts.Count; i++)
//{
// if (clusterWts[i] == null) continue;
// for (int c = 0; c < space; c++)
// {
// col++;
// //for (int j = 0; j < clusterSpectrogram.GetLength(1); j++) clusterSpectrogram[col, j] = clusterWts.Count+3;
// for (int j = 0; j < clusterWts[i].Length; j++)
// {
// if (clusterWts[i][j] > 0.0) clusterSpectrogram[col, excludeBins + j - 1] = i + 1;
// }
// }
// //col += 2;
//}
return(clusterSpectrogram);
}
