private void ClusteringParamsGetHandler(rosapi.GetParamResponse res)
{
Debug.Log("clustering");
Debug.Log(res.value);
parameters = JsonSerializer.Deserialize <ClusteringParameters>(res.value);
ClusterTolerance = parameters.cluster_tolerance;
MinClusterSize = parameters.min_cluster_size;
}
private SetOfClusters CreateDefaultSetOfClusters()
{
var clusteringParams = new ClusteringParameters()
{
MaxNumberDisagreements = 0, MinNumberAgreements = 0
};
var setOfClusters = new SetOfClusters(clusteringParams);
Assert.Equal(0, setOfClusters.Clusters.Count());
return(setOfClusters);
}
public void ClusteringStrategyTest()
{
var data = CreateMarketData();
var parameters = new ClusteringParameters();
var target = SimulateStrategy(
data,
x => x.Create(parameters),
true);
var actual = ToApprovedString(target);
Approvals.Verify(actual);
}
public ClusteringStrategy(
ISearcher searcher,
IMarketDataCache marketDataCache,
RatingService ratingService,
IStakingService stakingService,
ClusteringParameters parameters)
{
_searcher = searcher;
_marketDataCache = marketDataCache;
_ratingService = ratingService;
_stakingService = stakingService;
_parameters = parameters;
}
public void Optimise(DateTime fromDate, DateTime toDate)
{
_stakingService.Evaluate(fromDate, toDate);
var history = _marketDataCache.TakeUntil(toDate).ToArray();
// Build 2d array where each cell is sized to has equal data points and probability
var xData = history.Select(_xSelector).ToArray();
var yData = history.Select(_ySelector).ToArray();
var grid = new Grid <List <decimal> >(xData, yData, _parameters.Partitions);
// Place each data point in appropriate cell
var marketDataValues = _ratingService.CalculateMarketDataValues(history);
foreach (var data in history)
{
var x = _xSelector(data);
var y = _ySelector(data);
var cell = grid[x, y];
marketDataValues.TryGetValue(data.Date, out var value);
cell.Add(value);
}
// Determine the average value of each cell
var averages = grid
.Where(x => x.Any())
.Select(GetAverage)
.Distinct()
.ToArray();
var minValue = Convert.ToInt32(Math.Round(averages.Min(), MidpointRounding.ToNegativeInfinity));
var maxValue = Convert.ToInt32(Math.Round(averages.Max(), MidpointRounding.ToPositiveInfinity));
var range = maxValue - minValue;
// Search for optimal buy threshold
var potentials = Enumerable.Range(minValue, range)
.SelectMany(t => Enumerable.Range(2, 8)
.Select(p => new ClusteringParameters
{
Partitions = p,
Threshold = -t,
Grid = grid
}));
var optimal = _searcher.Maximum(potentials, fromDate, toDate);
_parameters = (ClusteringParameters)optimal;
}
public IStrategy Create(IParameters parameters)
{
return(parameters switch
{
LinearRegressionParameters p => Create(p),
RelativeStrengthParameters p => Create(p),
DeltaParameters p => Create(p),
VolumeParameters p => Create(p),
GradientParameters p => Create(p),
EntropyParameters p => Create(p),
StaticDatesParameters p => Create(p),
MovingAverageParameters p => Create(p),
HolidayEffectParameters p => Create(p),
WeightedParameters p => Create(p),
OptimalStoppingParameters p => Create(p),
ProbabilityParameters p => Create(p),
SpreadParameters p => Create(p),
ClusteringParameters p => Create(p),
_ => throw new NotImplementedException(),
});
/// <summary>
/// This CLUSTERING method is called only from IndexCalculate.cs
/// and TESTMETHOD_SpectralClustering(string wavFilePath, string outputDir, int frameSize)
/// It estimates the number of spectral clusters in a spectrogram,
/// and outputs two summary indices: cluster count (also called spectral diversity) and the threegram count.
/// IMPORTANT NOTE: The passed spectrogram MUST be already noise reduced.
/// This clustering algorithm is a highly reduced version of binary ART, Adaptive resonance Theory, designed for speed.
/// </summary>
/// <param name="spectrogram">a collection of spectra that are to be clustered</param>
/// <param name="lowerBinBound">lower end of the bird-band</param>
/// <param name="upperBinBound">upper end of the bird-band</param>
/// <param name="binaryThreshold">used to convert real value spectrum to binary</param>
public static ClusterInfo ClusterTheSpectra(double[,] spectrogram, int lowerBinBound, int upperBinBound, double binaryThreshold)
{
// Use verbose only when debugging
// SpectralClustering.Verbose = true;
var parameters = new ClusteringParameters(lowerBinBound, upperBinBound, binaryThreshold, DefaultRowSumThreshold);
TrainingDataInfo data = GetTrainingDataForClustering(spectrogram, parameters);
// cluster pruning parameters
const double weightThreshold = DefaultRowSumThreshold; // used to remove weight vectors whose sum of wts <= threshold
const int hitThreshold = DefaultHitThreshold; // used to remove weight vectors which have fewer than the threshold number of hits i.e. sets minimum cluster size
// Skip clustering if not enough suitable training data
if (data.TrainingData.Count <= 8)
{
int freqBinCount = spectrogram.GetLength(1);
return(new ClusterInfo(freqBinCount));
}
var clusterInfo = ClusterAnalysis(data.TrainingData, weightThreshold, hitThreshold, data.SelectedFrames);
return(clusterInfo);
}
private void ExecuteClusteringTest(List <VeadGroup> groups, List <List <VeadGroup> > expectedClusters, List <string> consensusSites, int?minNumAgreements = null, int?maxNumDisagreements = null, int ploidyConstraint = -1)
{
var options = new ClusteringParameters();
if (minNumAgreements != null)
{
options.MinNumberAgreements = (int)minNumAgreements;
}
if (maxNumDisagreements != null)
{
options.MaxNumberDisagreements = (int)maxNumDisagreements;
}
options.ClusterConstraint = ploidyConstraint;
var clusterer = new NeighborhoodClusterer(options);
var clusterSet = clusterer.ClusterVeadGroups(groups, "ndbdID");
Assert.Equal(expectedClusters.Count, clusterSet.NumClusters);
foreach (var cluster in clusterSet.Clusters)
{
var clusterConsensus = string.Join(",", cluster.GetConsensusSites().ToList());
Assert.Equal(1, consensusSites.Count(c => c == clusterConsensus));
}
}
public SetOfClusters(ClusteringParameters clusterParams)
{
_clusterParameters = clusterParams;
}
/// <summary>
/// First convert spectrogram to Binary using threshold. An amplitude threshold of 0.03 = -30 dB. An amplitude threhold of 0.05 = -26dB.
/// </summary>
public static TrainingDataInfo GetTrainingDataForClustering(double[,] spectrogram, ClusteringParameters cp)
{
int frameCount = spectrogram.GetLength(0);
int freqBinCount = spectrogram.GetLength(1);
var trainingData = new List <double[]>(); // training data that will be used for clustering
double[,] trainingDataAsSpectrogram = new double[frameCount, freqBinCount];
// training data represented in spectrogram
bool[] selectedFrames = new bool[frameCount];
for (int r = 0; r < frameCount; r++)
{
double[] spectrum = DataTools.GetRow(spectrogram, r);
spectrum = DataTools.VectorReduceLength(spectrum, cp.ReductionFactor);
// reduce length of the vector by factor of N and convert to binary
for (int i = 0; i < spectrum.Length; i++)
{
if (spectrum[i] >= cp.IntensityThreshold)
{
spectrum[i] = 1.0;
}
else
{
spectrum[i] = 0.0;
}
}
// remove isolated peaks.
//for (int i = 1; i < spectrum.Length - 1; i++)
//{
// if ((spectrum[i] == 1.0) && (spectrum[i - 1] == 0.0) && (spectrum[i + 1] == 0.0))
// {
// spectrum[i] = 0.0;
// }
//}
// only include frames where activity exceeds threshold
if (spectrum.Sum() > cp.RowSumThreshold)
{
trainingData.Add(spectrum);
IncludeSpectrumInSpectrogram(trainingDataAsSpectrogram, r, spectrum, cp.ReductionFactor);
selectedFrames[r] = true;
}
}
return(new TrainingDataInfo(trainingDataAsSpectrogram, trainingData, selectedFrames, cp.LowBinBound, cp.UpperBinBound, cp.IntensityThreshold));
}
/// <summary>
/// This method was set up as a TESTMETHOD in May 2017 but has not yet been debugged.
/// It was transferred from Sandpit.cls. It is several years old.
/// Updated May 2017.
/// </summary>
public static void TESTMETHOD_SpectralClustering(string wavFilePath, string outputDir, int frameSize)
{
string imageViewer = @"C:\Windows\system32\mspaint.exe";
var recording = new AudioRecording(wavFilePath); // get recording segment
// test clustering using amplitude spectrogram
// double binaryThreshold = 0.07; // An amplitude threshold of 0.03 = -30 dB. A threshold of 0.05 = -26dB.
// double[,] spectrogramData = GetAmplitudeSpectrogramNoiseReduced(recording, frameSize);
// test clustering using decibel spectrogram
double binaryThreshold = DefaultBinaryThresholdInDecibels; // A decibel threshold for converting to binary
double[,] spectrogramData = GetDecibelSpectrogramNoiseReduced(recording, frameSize);
//#######################################################################################################################################
int nyquistFreq = recording.Nyquist;
int frameCount = spectrogramData.GetLength(0);
int freqBinCount = spectrogramData.GetLength(1);
double binWidth = nyquistFreq / (double)freqBinCount;
// We only use the midband of the Spectrogram, i.e. the band between lowerBinBound and upperBinBound.
// int lowFreqBound = 1000;
// int upperBinBound = freqBinCount - 5;
// In June 2016, the mid-band was set to lowerBound=1000Hz, upperBound=8000hz, because this band contains most bird activity, i.e. it is the Bird-Band
// This was done in order to make the cluster summary indices more reflective of bird call activity.
// int lowerFreqBound = 482;
int lowerFreqBound = 1000;
int lowerBinBound = (int)Math.Ceiling(lowerFreqBound / binWidth);
int upperFreqBound = 8000;
int upperBinBound = (int)Math.Ceiling(upperFreqBound / binWidth);
var midBandSpectrogram = MatrixTools.Submatrix(spectrogramData, 0, lowerBinBound, spectrogramData.GetLength(0) - 1, upperBinBound);
var clusterInfo = ClusterTheSpectra(midBandSpectrogram, lowerBinBound, upperBinBound, binaryThreshold);
// transfer cluster info to spectral index results
var clusterSpectrum1 = RestoreFullLengthSpectrum(clusterInfo.ClusterSpectrum, freqBinCount, lowerBinBound);
Console.WriteLine("Lower BinBound=" + lowerBinBound);
Console.WriteLine("Upper BinBound=" + upperBinBound);
Console.WriteLine("Binary Threshold=" + binaryThreshold);
Console.WriteLine("Cluster Count =" + clusterInfo.ClusterCount);
Console.WriteLine("Three Gram Count=" + clusterInfo.TriGramUniqueCount);
// ###################################################################################
// Now repeat the entire process again. This time we want to get intermediate results to superimpose on spectrogram
// Need to specify additional parameters. ACTIVITY THRESHOLD - require activity in at least N bins to include spectrum for training
double rowSumThreshold = DefaultRowSumThreshold;
var parameters = new ClusteringParameters(lowerBinBound, upperBinBound, binaryThreshold, rowSumThreshold);
TrainingDataInfo data = GetTrainingDataForClustering(midBandSpectrogram, parameters);
// make a normal standard decibel spectogram on which to superimpose cluster results
var stdSpectrogram = GetStandardSpectrogram(recording, frameSize);
var image = DrawSonogram(stdSpectrogram, null, null, 0.0, null);
SaveAndViewSpectrogramImage(image, outputDir, "test0Spectrogram.png", imageViewer);
// Debug.Assert(data.TrainingDataAsSpectrogram != null, "data.TrainingDataAsSpectrogram != null");
double[,] overlay = ConvertOverlayToSpectrogramSize(data.TrainingDataAsSpectrogram, lowerBinBound, frameCount, freqBinCount);
image = DrawSonogram(stdSpectrogram, null, null, 0.0, overlay);
SaveAndViewSpectrogramImage(image, outputDir, "test1Spectrogram.png", imageViewer);
// Return if no suitable training data for clustering
if (data.TrainingData.Count <= 8)
{
Console.WriteLine("Abort clustering. Only {0} spectra available for training data. Must be at least 9.", data.TrainingData.Count);
}
else
{
// the following are pruning parameters
double wtThreshold = rowSumThreshold; // used to remove wt vectors whose sum of wts <= threshold
int hitThreshold = DefaultHitThreshold; // used to remove wt vectors which have < N hits, i.e. cluster size < N
clusterInfo = ClusterAnalysis(data.TrainingData, wtThreshold, hitThreshold, data.SelectedFrames);
Console.WriteLine("Binary Threshold=" + binaryThreshold);
Console.WriteLine("Weight Threshold=" + wtThreshold);
Console.WriteLine("Hit Threshold=" + hitThreshold);
Console.WriteLine("Cluster Count=" + clusterInfo.ClusterCount);
Console.WriteLine("Three Gram Count=" + clusterInfo.TriGramUniqueCount);
image = DrawClusterSpectrogram(stdSpectrogram, clusterInfo, data, lowerBinBound);
SaveAndViewSpectrogramImage(image, outputDir, "test2Spectrogram.png", imageViewer);
// transfer cluster info to spectral index results
var clusterSpectrum2 = RestoreFullLengthSpectrum(clusterInfo.ClusterSpectrum, freqBinCount, lowerBinBound);
TestTools.CompareTwoArrays(clusterSpectrum1, clusterSpectrum2);
}
}
public NeighborhoodClusterer(ClusteringParameters options)
{
_options = options;
}
public NeighborhoodClusterer(ClusteringParameters options, bool debug)
{
_options = options;
_debug = debug;
}
