public void LinearFrequencyScale()
{
var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav");
var opFileStem = "BAC2_20071008";
var outputDir = this.outputDirectory;
var outputImagePath = Path.Combine(outputDir.FullName, "LinearScaleSonogram.png");
var recording = new AudioRecording(recordingPath);
// specfied linear scale
int nyquist = 11025;
int frameSize = 1024;
int hertzInterval = 1000;
var freqScale = new FrequencyScale(nyquist, frameSize, hertzInterval);
var fst = freqScale.ScaleType;
var sonoConfig = new SonogramConfig
{
WindowSize = freqScale.FinalBinCount * 2,
WindowOverlap = 0.2,
SourceFName = recording.BaseName,
NoiseReductionType = NoiseReductionType.None,
NoiseReductionParameter = 0.0,
};
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// DO NOISE REDUCTION
var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data);
sonogram.Data = dataMatrix;
sonogram.Configuration.WindowSize = freqScale.WindowSize;
var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
image.Save(outputImagePath, ImageFormat.Png);
// DO FILE EQUALITY TEST
var stemOfExpectedFile = opFileStem + "_LinearScaleGridLineLocations.EXPECTED.json";
var stemOfActualFile = opFileStem + "_LinearScaleGridLineLocations.ACTUAL.json";
// Check that freqScale.GridLineLocations are correct
var expectedFile1 = PathHelper.ResolveAsset("FrequencyScale\\" + stemOfExpectedFile);
if (!expectedFile1.Exists)
{
LoggedConsole.WriteErrorLine("An EXPECTED results file does not exist. Test will fail!");
LoggedConsole.WriteErrorLine(
$"If ACTUAL results file is correct, move it to dir `{PathHelper.TestResources}` and change its suffix to <.EXPECTED.json>");
}
var resultFile1 = new FileInfo(Path.Combine(outputDir.FullName, stemOfActualFile));
Json.Serialise(resultFile1, freqScale.GridLineLocations);
FileEqualityHelpers.TextFileEqual(expectedFile1, resultFile1);
// Check that image dimensions are correct
Assert.AreEqual(566, image.Height);
Assert.AreEqual(1621, image.Width);
}
public void LinearFrequencyScaleDefault()
{
// relative path because post-Build command transfers files to ...\\Work\GitHub\...\bin\Debug subfolder.
var recordingPath = @"Recordings\BAC2_20071008-085040.wav";
var opFileStem = "BAC2_20071008";
var outputDir = this.outputDirectory;
var outputImagePath = Path.Combine(outputDir.FullName, "DefaultLinearScaleSonogram.png");
var recording = new AudioRecording(recordingPath);
// default linear scale
var fst = FreqScaleType.Linear;
var freqScale = new FrequencyScale(fst);
var sonoConfig = new SonogramConfig
{
WindowSize = freqScale.FinalBinCount * 2,
WindowOverlap = 0.2,
SourceFName = recording.BaseName,
NoiseReductionType = NoiseReductionType.None,
NoiseReductionParameter = 0.0,
};
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
sonogram.Configuration.WindowSize = freqScale.WindowSize;
// DO NOISE REDUCTION
var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data);
sonogram.Data = dataMatrix;
var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
image.Save(outputImagePath, ImageFormat.Png);
// DO UNIT TESTING
var stemOfExpectedFile = opFileStem + "_DefaultLinearScaleGridLineLocations.EXPECTED.json";
var stemOfActualFile = opFileStem + "_DefaultLinearScaleGridLineLocations.ACTUAL.json";
// Check that freqScale.GridLineLocations are correct
var expectedFile1 = new FileInfo("FrequencyScale\\" + stemOfExpectedFile);
if (!expectedFile1.Exists)
{
LoggedConsole.WriteErrorLine("An EXPECTED results file does not exist. Test will fail!");
LoggedConsole.WriteErrorLine("If ACTUAL results file is correct, move it to dir <...\\TestResources\\FrequencyScale> and change its suffix to <.EXPECTED.json>");
}
var resultFile1 = new FileInfo(Path.Combine(outputDir.FullName, stemOfActualFile));
Json.Serialise(resultFile1, freqScale.GridLineLocations);
FileEqualityHelpers.TextFileEqual(expectedFile1, resultFile1);
// Check that image dimensions are correct
Assert.AreEqual(310, image.Height);
Assert.AreEqual(3247, image.Width);
}
public void LinearFrequencyScale()
{
var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav");
var outputImagePath = this.outputDirectory.CombineFile("DefaultLinearScaleSonogram.png");
var recording = new AudioRecording(recordingPath);
// specfied linear scale
int nyquist = 11025;
int frameSize = 1024;
int hertzInterval = 1000;
var freqScale = new FrequencyScale(nyquist, frameSize, hertzInterval);
var fst = freqScale.ScaleType;
var sonoConfig = new SonogramConfig
{
WindowSize = freqScale.FinalBinCount * 2,
WindowOverlap = 0.2,
SourceFName = recording.BaseName,
NoiseReductionType = NoiseReductionType.None,
NoiseReductionParameter = 0.0,
};
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// DO NOISE REDUCTION
var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data);
sonogram.Data = dataMatrix;
sonogram.Configuration.WindowSize = freqScale.WindowSize;
var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst, freqScale.GridLineLocations);
image.Save(outputImagePath);
var expected = new[, ]
{
{ 46, 1000 },
{ 92, 2000 },
{ 139, 3000 },
{ 185, 4000 },
{ 232, 5000 },
{ 278, 6000 },
{ 325, 7000 },
{ 371, 8000 },
{ 417, 9000 },
{ 464, 10000 },
{ 510, 11000 },
};
Assert.That.MatricesAreEqual(expected, freqScale.GridLineLocations);
// Check that image dimensions are correct
Assert.AreEqual(566, image.Height);
Assert.AreEqual(1621, image.Width);
}
public void LinearFrequencyScaleDefault()
{
var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav");
var outputImagePath = this.outputDirectory.CombineFile("DefaultLinearScaleSonogram.png");
var recording = new AudioRecording(recordingPath);
// default linear scale
var fst = FreqScaleType.Linear;
var freqScale = new FrequencyScale(fst);
var sonoConfig = new SonogramConfig
{
WindowSize = freqScale.FinalBinCount * 2,
WindowOverlap = 0.2,
SourceFName = recording.BaseName,
NoiseReductionType = NoiseReductionType.None,
NoiseReductionParameter = 0.0,
};
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
sonogram.Configuration.WindowSize = freqScale.WindowSize;
// DO NOISE REDUCTION
var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data);
sonogram.Data = dataMatrix;
var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst, freqScale.GridLineLocations);
image.Save(outputImagePath);
// Check that freqScale.GridLineLocations are correct
var expected = new[, ]
{
{ 23, 1000 },
{ 46, 2000 },
{ 69, 3000 },
{ 92, 4000 },
{ 116, 5000 },
{ 139, 6000 },
{ 162, 7000 },
{ 185, 8000 },
{ 208, 9000 },
{ 232, 10000 },
{ 255, 11000 },
};
Assert.That.MatricesAreEqual(expected, freqScale.GridLineLocations);
// Check that image dimensions are correct
Assert.AreEqual(310, image.Height);
Assert.AreEqual(3247, image.Width);
}
/// <summary>
/// METHOD TO CHECK IF SPECIFIED linear FREQ SCALE IS WORKING
/// Check it on standard one minute recording.
/// </summary>
public static void TESTMETHOD_LinearFrequencyScale()
{
var recordingPath = @"C:\SensorNetworks\SoftwareTests\TestRecordings\BAC2_20071008-085040.wav";
var outputDir = @"C:\SensorNetworks\SoftwareTests\TestFrequencyScale".ToDirectoryInfo();
var expectedResultsDir = Path.Combine(outputDir.FullName, TestTools.ExpectedResultsDir).ToDirectoryInfo();
var outputImagePath = Path.Combine(outputDir.FullName, "linearScaleSonogram.png");
var opFileStem = "BAC2_20071008";
var recording = new AudioRecording(recordingPath);
// specfied linear scale
int nyquist = 11025;
int frameSize = 1024;
int hertzInterval = 1000;
var freqScale = new FrequencyScale(nyquist, frameSize, hertzInterval);
var fst = freqScale.ScaleType;
var sonoConfig = new SonogramConfig
{
WindowSize = freqScale.FinalBinCount * 2,
WindowOverlap = 0.2,
SourceFName = recording.BaseName,
//NoiseReductionType = NoiseReductionType.Standard,
NoiseReductionType = NoiseReductionType.None,
NoiseReductionParameter = 0.0,
};
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// DO NOISE REDUCTION
var dataMatrix = SNR.NoiseReduce_Standard(sonogram.Data);
sonogram.Data = dataMatrix;
sonogram.Configuration.WindowSize = freqScale.WindowSize;
var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
image.Save(outputImagePath);
// DO FILE EQUALITY TEST
string testName = "testName";
var expectedTestFile = new FileInfo(Path.Combine(expectedResultsDir.FullName, "FrequencyLinearScaleTest.EXPECTED.json"));
var resultFile = new FileInfo(Path.Combine(outputDir.FullName, opFileStem + "FrequencyLinearScaleTestResults.json"));
Acoustics.Shared.Csv.Csv.WriteMatrixToCsv(resultFile, freqScale.GridLineLocations);
TestTools.FileEqualityTest(testName, resultFile, expectedTestFile);
LoggedConsole.WriteLine("Completed Linear Frequency Scale test");
Console.WriteLine("\n\n");
}
/// <summary>
/// METHOD TO CHECK IF SPECIFIED MEL FREQ SCALE IS WORKING
/// Check it on standard one minute recording.
/// </summary>
public static void TESTMETHOD_MelFrequencyScale()
{
var recordingPath = @"C:\SensorNetworks\SoftwareTests\TestRecordings\BAC2_20071008-085040.wav";
var outputDir = @"C:\SensorNetworks\SoftwareTests\TestFrequencyScale".ToDirectoryInfo();
var expectedResultsDir = Path.Combine(outputDir.FullName, TestTools.ExpectedResultsDir).ToDirectoryInfo();
var outputImagePath = Path.Combine(outputDir.FullName, "melScaleSonogram.png");
var opFileStem = "BAC2_20071008";
var recording = new AudioRecording(recordingPath);
int nyquist = recording.Nyquist;
int frameSize = 1024;
int finalBinCount = 256;
int hertzInterval = 1000;
FreqScaleType scaleType = FreqScaleType.Mel;
var freqScale = new FrequencyScale(scaleType, nyquist, frameSize, finalBinCount, hertzInterval);
var fst = freqScale.ScaleType;
var sonoConfig = new SonogramConfig
{
WindowSize = frameSize,
WindowOverlap = 0.2,
SourceFName = recording.BaseName,
DoMelScale = (scaleType == FreqScaleType.Mel) ? true : false,
MelBinCount = (scaleType == FreqScaleType.Mel) ? finalBinCount : frameSize / 2,
//NoiseReductionType = NoiseReductionType.Standard,
NoiseReductionType = NoiseReductionType.None,
NoiseReductionParameter = 0.0,
};
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// DRAW SPECTROGRAM
var image = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
image.Save(outputImagePath);
// DO FILE EQUALITY TEST
string testName = "MelTest";
var expectedTestFile = new FileInfo(Path.Combine(expectedResultsDir.FullName, "MelFrequencyScaleTest.EXPECTED.json"));
var resultFile = new FileInfo(Path.Combine(outputDir.FullName, opFileStem + "MelFrequencyLinearScaleTestResults.json"));
Acoustics.Shared.Csv.Csv.WriteMatrixToCsv(resultFile, freqScale.GridLineLocations);
TestTools.FileEqualityTest(testName, resultFile, expectedTestFile);
LoggedConsole.WriteLine("Completed Mel Frequency Scale test");
Console.WriteLine("\n\n");
}
public void TestPcaWhitening()
{
var recordingPath = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav");
var outputDir = this.outputDirectory;
var outputImagePath = Path.Combine(outputDir.FullName, "ReconstrcutedSpectrogram.png");
var fst = FreqScaleType.Linear;
var freqScale = new FrequencyScale(fst);
var recording = new AudioRecording(recordingPath);
var sonoConfig = new SonogramConfig
{
WindowSize = freqScale.FinalBinCount * 2,
WindowOverlap = 0.2,
SourceFName = recording.BaseName,
NoiseReductionType = NoiseReductionType.None,
NoiseReductionParameter = 0.0,
};
// GENERATE AMPLITUDE SPECTROGRAM
var spectrogram = new AmplitudeSonogram(sonoConfig, recording.WavReader);
spectrogram.Configuration.WindowSize = freqScale.WindowSize;
// DO RMS NORMALIZATION
spectrogram.Data = SNR.RmsNormalization(spectrogram.Data);
// CONVERT NORMALIZED AMPLITUDE SPECTROGRAM TO dB SPECTROGRAM
var sonogram = new SpectrogramStandard(spectrogram);
// DO NOISE REDUCTION
var dataMatrix = PcaWhitening.NoiseReduction(sonogram.Data);
sonogram.Data = dataMatrix;
// Do Patch Sampling
int rows = sonogram.Data.GetLength(0);
int columns = sonogram.Data.GetLength(1);
int patchWidth = columns;
int patchHeight = 1;
int numberOfPatches = (rows / patchHeight) * (columns / patchWidth);
var sequentialPatches = PatchSampling.GetPatches(sonogram.Data, patchWidth, patchHeight, numberOfPatches, PatchSampling.SamplingMethod.Sequential);
double[,] sequentialPatchMatrix = sequentialPatches.ToMatrix();
// DO PCA WHITENING
var whitenedSpectrogram = PcaWhitening.Whitening(sequentialPatchMatrix);
// reconstructing the spectrogram from sequential patches and the projection matrix obtained from random patches
var projectionMatrix = whitenedSpectrogram.ProjectionMatrix;//whitenedSpectrogram.projectionMatrix;
var eigenVectors = whitenedSpectrogram.EigenVectors;
var numComponents = whitenedSpectrogram.Components;
double[,] reconstructedSpec = PcaWhitening.ReconstructSpectrogram(projectionMatrix, sequentialPatchMatrix, eigenVectors, numComponents);
sonogram.Data = PatchSampling.ConvertPatches(reconstructedSpec, patchWidth, patchHeight, columns);
var reconstructedSpecImage = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "RECONSTRUCTEDSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
reconstructedSpecImage.Save(outputImagePath, ImageFormat.Png);
// DO UNIT TESTING
Assert.AreEqual(spectrogram.Data.GetLength(0), sonogram.Data.GetLength(0));
Assert.AreEqual(spectrogram.Data.GetLength(1), sonogram.Data.GetLength(1));
}
public static AudioToSonogramResult GenerateFourSpectrogramImages(
FileInfo sourceRecording,
FileInfo path2SoxSpectrogram,
Dictionary <string, string> configDict,
bool dataOnly = false,
bool makeSoxSonogram = false)
{
var result = new AudioToSonogramResult();
if (dataOnly && makeSoxSonogram)
{
throw new ArgumentException("Can't produce data only for a SoX sonogram");
}
if (makeSoxSonogram)
{
SpectrogramTools.MakeSonogramWithSox(sourceRecording, configDict, path2SoxSpectrogram);
result.Path2SoxImage = path2SoxSpectrogram;
}
else if (dataOnly)
{
var recordingSegment = new AudioRecording(sourceRecording.FullName);
var sonoConfig = new SonogramConfig(configDict); // default values config
// disable noise removal
sonoConfig.NoiseReductionType = NoiseReductionType.None;
Log.Warn("Noise removal disabled!");
var sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
result.DecibelSpectrogram = sonogram;
}
else
{
// init the image stack
var list = new List <Image>();
// IMAGE 1) draw amplitude spectrogram
var recordingSegment = new AudioRecording(sourceRecording.FullName);
var sonoConfig = new SonogramConfig(configDict); // default values config
// disable noise removal for first two spectrograms
var disabledNoiseReductionType = sonoConfig.NoiseReductionType;
sonoConfig.NoiseReductionType = NoiseReductionType.None;
BaseSonogram sonogram = new AmplitudeSonogram(sonoConfig, recordingSegment.WavReader);
// remove the DC bin if it has not already been removed.
// Assume test of divisible by 2 is good enough.
int binCount = sonogram.Data.GetLength(1);
if (!binCount.IsEven())
{
sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.FrameCount - 1, binCount - 1);
}
//save spectrogram data at this point - prior to noise reduction
var spectrogramDataBeforeNoiseReduction = sonogram.Data;
const double neighbourhoodSeconds = 0.25;
int neighbourhoodFrames = (int)(sonogram.FramesPerSecond * neighbourhoodSeconds);
const double lcnContrastLevel = 0.001;
LoggedConsole.WriteLine("LCN: FramesPerSecond (Prior to LCN) = {0}", sonogram.FramesPerSecond);
LoggedConsole.WriteLine("LCN: Neighbourhood of {0} seconds = {1} frames", neighbourhoodSeconds, neighbourhoodFrames);
const int lowPercentile = 20;
sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLowestPercentileSubtraction(sonogram.Data, lowPercentile);
sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLCNDivision(sonogram.Data, neighbourhoodFrames, lcnContrastLevel);
//sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLowestPercentileSubtraction(sonogram.Data, lowPercentile);
var image = sonogram.GetImageFullyAnnotated("AMPLITUDE SPECTROGRAM + Bin LCN (Local Contrast Normalisation)");
list.Add(image);
//string path2 = @"C:\SensorNetworks\Output\Sonograms\dataInput2.png";
//Histogram.DrawDistributionsAndSaveImage(sonogram.Data, path2);
// double[,] matrix = sonogram.Data;
double[,] matrix = ImageTools.WienerFilter(sonogram.Data, 3);
double ridgeThreshold = 0.25;
byte[,] hits = RidgeDetection.Sobel5X5RidgeDetectionExperiment(matrix, ridgeThreshold);
hits = RidgeDetection.JoinDisconnectedRidgesInMatrix(hits, matrix, ridgeThreshold);
image = SpectrogramTools.CreateFalseColourAmplitudeSpectrogram(spectrogramDataBeforeNoiseReduction, null, hits);
image = sonogram.GetImageAnnotatedWithLinearHerzScale(image, "AMPLITUDE SPECTROGRAM + LCN + ridge detection");
list.Add(image);
Image envelopeImage = ImageTrack.DrawWaveEnvelopeTrack(recordingSegment, image.Width);
list.Add(envelopeImage);
// IMAGE 2) now draw the standard decibel spectrogram
sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
result.DecibelSpectrogram = (SpectrogramStandard)sonogram;
image = sonogram.GetImageFullyAnnotated("DECIBEL SPECTROGRAM");
list.Add(image);
Image segmentationImage = ImageTrack.DrawSegmentationTrack(
sonogram,
EndpointDetectionConfiguration.K1Threshold,
EndpointDetectionConfiguration.K2Threshold,
image.Width);
list.Add(segmentationImage);
// keep the sonogram data for later use
double[,] dbSpectrogramData = (double[, ])sonogram.Data.Clone();
// 3) now draw the noise reduced decibel spectrogram
// #NOISE REDUCTION PARAMETERS - restore noise reduction ##################################################################
sonoConfig.NoiseReductionType = disabledNoiseReductionType;
sonoConfig.NoiseReductionParameter = double.Parse(configDict[AnalysisKeys.NoiseBgThreshold] ?? "2.0");
// #NOISE REDUCTION PARAMETERS - MARINE HACK ##################################################################
//sonoConfig.NoiseReductionType = NoiseReductionType.FIXED_DYNAMIC_RANGE;
//sonoConfig.NoiseReductionParameter = 80.0;
sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
image = sonogram.GetImageFullyAnnotated("DECIBEL SPECTROGRAM + Lamel noise subtraction");
list.Add(image);
// keep the sonogram data for later use
double[,] nrSpectrogramData = sonogram.Data;
// 4) A FALSE-COLOUR VERSION OF SPECTROGRAM
// ########################### SOBEL ridge detection
ridgeThreshold = 3.5;
matrix = ImageTools.WienerFilter(dbSpectrogramData, 3);
hits = RidgeDetection.Sobel5X5RidgeDetectionExperiment(matrix, ridgeThreshold);
// ########################### EIGEN ridge detection
//double ridgeThreshold = 6.0;
//double dominanceThreshold = 0.7;
//var rotatedData = MatrixTools.MatrixRotate90Anticlockwise(dbSpectrogramData);
//byte[,] hits = RidgeDetection.StructureTensorRidgeDetection(rotatedData, ridgeThreshold, dominanceThreshold);
//hits = MatrixTools.MatrixRotate90Clockwise(hits);
// ########################### EIGEN ridge detection
image = SpectrogramTools.CreateFalseColourDecibelSpectrogram(dbSpectrogramData, nrSpectrogramData, hits);
image = sonogram.GetImageAnnotatedWithLinearHerzScale(image, "DECIBEL SPECTROGRAM - Colour annotated");
list.Add(image);
// 5) TODO: ONE OF THESE YEARS FIX UP THE CEPTRAL SONOGRAM
////SpectrogramCepstral cepgram = new SpectrogramCepstral((AmplitudeSonogram)amplitudeSpg);
////var mti3 = SpectrogramTools.Sonogram2MultiTrackImage(sonogram, configDict);
////var image3 = mti3.GetImage();
////image3.Save(fiImage.FullName + "3", ImageFormat.Png);
// 6) COMBINE THE SPECTROGRAM IMAGES
result.CompositeImage = ImageTools.CombineImagesVertically(list);
}
return(result);
}
/// <summary>
/// Calculates the following spectrograms as per settings in the Images array in the config file: Towsey.SpectrogramGenerator.yml:
/// Waveform.
/// DecibelSpectrogram.
/// DecibelSpectrogramNoiseReduced.
/// CepstralSpectrogram.
/// DifferenceSpectrogram.
/// AmplitudeSpectrogramLocalContrastNormalization.
/// Experimental.
/// Comment the config.yml file with a hash, those spectrograms that are not required.
/// </summary>
/// <param name="sourceRecording">The name of the original recording.</param>
/// <param name="config">Contains parameter info to make spectrograms.</param>
/// <param name="sourceRecordingName">.Name of source recording. Required only spectrogram labels.</param>
public static AudioToSonogramResult GenerateSpectrogramImages(
FileInfo sourceRecording,
SpectrogramGeneratorConfig config,
string sourceRecordingName)
{
//int signalLength = recordingSegment.WavReader.GetChannel(0).Length;
var recordingSegment = new AudioRecording(sourceRecording.FullName);
int sampleRate = recordingSegment.WavReader.SampleRate;
var result = new AudioToSonogramResult();
var requestedImageTypes = config.Images ?? new[] { SpectrogramImageType.DecibelSpectrogram };
var @do = requestedImageTypes.ToHashSet();
int frameSize = config.GetIntOrNull("FrameLength") ?? 512;
int frameStep = config.GetIntOrNull("FrameStep") ?? 441;
// must calculate this because used later on.
double frameOverlap = (frameSize - frameStep) / (double)frameSize;
// Default noiseReductionType = Standard
var bgNoiseThreshold = config.BgNoiseThreshold;
// threshold for drawing the difference spectrogram
var differenceThreshold = config.DifferenceThreshold;
// EXTRACT ENVELOPE and SPECTROGRAM FROM RECORDING SEGMENT
var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recordingSegment, frameSize, frameStep);
var sonoConfig = new SonogramConfig()
{
epsilon = recordingSegment.Epsilon,
SampleRate = sampleRate,
WindowSize = frameSize,
WindowStep = frameStep,
WindowOverlap = frameOverlap,
WindowPower = dspOutput1.WindowPower,
Duration = recordingSegment.Duration,
NoiseReductionType = NoiseReductionType.Standard,
NoiseReductionParameter = bgNoiseThreshold,
};
var images = new Dictionary <SpectrogramImageType, Image <Rgb24> >(requestedImageTypes.Length);
// IMAGE 1) draw the WAVEFORM
if (@do.Contains(Waveform))
{
var minValues = dspOutput1.MinFrameValues;
var maxValues = dspOutput1.MaxFrameValues;
int height = config.WaveformHeight;
var waveformImage = GetWaveformImage(minValues, maxValues, height);
// add in the title bar and time scales.
string title =
$"WAVEFORM - {sourceRecordingName} (min value={dspOutput1.MinSignalValue:f3}, max value={dspOutput1.MaxSignalValue:f3})";
var titleBar = BaseSonogram.DrawTitleBarOfGrayScaleSpectrogram(
title,
waveformImage.Width,
ImageTags[Waveform]);
var startTime = TimeSpan.Zero;
var xAxisTicInterval = TimeSpan.FromSeconds(1);
TimeSpan xAxisPixelDuration = TimeSpan.FromSeconds(frameStep / (double)sampleRate);
var labelInterval = TimeSpan.FromSeconds(5);
waveformImage = BaseSonogram.FrameSonogram(
waveformImage,
titleBar,
startTime,
xAxisTicInterval,
xAxisPixelDuration,
labelInterval);
images.Add(Waveform, waveformImage);
}
// Draw various decibel spectrograms
var decibelTypes = new[] { SpectrogramImageType.DecibelSpectrogram, DecibelSpectrogramNoiseReduced, DifferenceSpectrogram, Experimental };
if (@do.Overlaps(decibelTypes))
{
// disable noise removal for first two spectrograms
var disabledNoiseReductionType = sonoConfig.NoiseReductionType;
sonoConfig.NoiseReductionType = NoiseReductionType.None;
//Get the decibel spectrogram
var decibelSpectrogram = new SpectrogramStandard(sonoConfig, dspOutput1.AmplitudeSpectrogram);
result.DecibelSpectrogram = decibelSpectrogram;
double[,] dbSpectrogramData = (double[, ])decibelSpectrogram.Data.Clone();
// IMAGE 2) Display the DecibelSpectrogram
if (@do.Contains(SpectrogramImageType.DecibelSpectrogram))
{
images.Add(
SpectrogramImageType.DecibelSpectrogram,
decibelSpectrogram.GetImageFullyAnnotated(
$"DECIBEL SPECTROGRAM ({sourceRecordingName})",
ImageTags[SpectrogramImageType.DecibelSpectrogram]));
}
if (@do.Overlaps(new[] { DecibelSpectrogramNoiseReduced, Experimental, CepstralSpectrogram }))
{
sonoConfig.NoiseReductionType = disabledNoiseReductionType;
sonoConfig.NoiseReductionParameter = bgNoiseThreshold;
double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram.Data);
decibelSpectrogram.Data =
SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram.Data, spectralDecibelBgn);
decibelSpectrogram.Data =
SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram.Data, nhThreshold: bgNoiseThreshold);
// IMAGE 3) DecibelSpectrogram - noise reduced
if (@do.Contains(DecibelSpectrogramNoiseReduced))
{
images.Add(
DecibelSpectrogramNoiseReduced,
decibelSpectrogram.GetImageFullyAnnotated(
$"DECIBEL SPECTROGRAM + Lamel noise subtraction. ({sourceRecordingName})",
ImageTags[DecibelSpectrogramNoiseReduced]));
}
// IMAGE 4) EXPERIMENTAL Spectrogram
if (@do.Contains(Experimental))
{
sonoConfig.NoiseReductionType = disabledNoiseReductionType;
images.Add(
Experimental,
GetDecibelSpectrogram_Ridges(
dbSpectrogramData,
decibelSpectrogram,
sourceRecordingName));
}
}
// IMAGE 5) draw difference spectrogram. This is derived from the original decibel spectrogram
if (@do.Contains(DifferenceSpectrogram))
{
//var differenceThreshold = configInfo.GetDoubleOrNull("DifferenceThreshold") ?? 3.0;
var differenceImage = GetDifferenceSpectrogram(dbSpectrogramData, differenceThreshold);
differenceImage = BaseSonogram.GetImageAnnotatedWithLinearHertzScale(
differenceImage,
sampleRate,
frameStep,
$"DECIBEL DIFFERENCE SPECTROGRAM ({sourceRecordingName})",
ImageTags[DifferenceSpectrogram]);
images.Add(DifferenceSpectrogram, differenceImage);
}
}
// IMAGE 6) Cepstral Spectrogram
if (@do.Contains(CepstralSpectrogram))
{
images.Add(
CepstralSpectrogram,
GetCepstralSpectrogram(sonoConfig, recordingSegment, sourceRecordingName));
}
// IMAGE 7) AmplitudeSpectrogram_LocalContrastNormalization
if (@do.Contains(AmplitudeSpectrogramLocalContrastNormalization))
{
var neighborhoodSeconds = config.NeighborhoodSeconds;
var lcnContrastParameter = config.LcnContrastLevel;
images.Add(
AmplitudeSpectrogramLocalContrastNormalization,
GetLcnSpectrogram(
sonoConfig,
recordingSegment,
sourceRecordingName,
neighborhoodSeconds,
lcnContrastParameter));
}
// now pick and combine images in order user specified
var sortedImages = requestedImageTypes.Select(x => images[x]);
// COMBINE THE SPECTROGRAM IMAGES
result.CompositeImage = ImageTools.CombineImagesVertically(sortedImages.ToArray());
return(result);
}
/// <summary>
/// Calculates the following spectrograms as per content of config.yml file:
/// Waveform: true.
/// DifferenceSpectrogram: true.
/// DecibelSpectrogram: true.
/// DecibelSpectrogram_NoiseReduced: true.
/// DecibelSpectrogram_Ridges: true.
/// AmplitudeSpectrogram_LocalContrastNormalization: true.
/// SoxSpectrogram: false.
/// Experimental: true.
/// </summary>
/// <param name="sourceRecording">The name of the original recording.</param>
/// <param name="configInfo">Contains parameter info to make spectrograms.</param>
/// <param name="sourceRecordingName">.Name of source recording. Required only spectrogram labels.</param>
public static AudioToSonogramResult GenerateSpectrogramImages(
FileInfo sourceRecording,
AnalyzerConfig configInfo,
string sourceRecordingName)
{
//int signalLength = recordingSegment.WavReader.GetChannel(0).Length;
var recordingSegment = new AudioRecording(sourceRecording.FullName);
int sampleRate = recordingSegment.WavReader.SampleRate;
var result = new AudioToSonogramResult();
// init the image stack
var list = new List <Image>();
bool doWaveForm = configInfo.GetBoolOrNull("Waveform") ?? false;
bool doDecibelSpectrogram = configInfo.GetBoolOrNull("DecibelSpectrogram") ?? false;
bool doNoiseReducedSpectrogram = configInfo.GetBoolOrNull("DecibelSpectrogram_NoiseReduced") ?? true;
bool doDifferenceSpectrogram = configInfo.GetBoolOrNull("DifferenceSpectrogram") ?? false;
bool doLcnSpectrogram = configInfo.GetBoolOrNull("AmplitudeSpectrogram_LocalContrastNormalization") ?? false;
bool doCepstralSpectrogram = configInfo.GetBoolOrNull("CepstralSpectrogram") ?? false;
bool doExperimentalSpectrogram = configInfo.GetBoolOrNull("Experimental") ?? false;
//Don't do SOX spectrogram.
//bool doSoxSpectrogram = configInfo.GetBool("SoxSpectrogram");
int frameSize = configInfo.GetIntOrNull("FrameLength") ?? 512;
int frameStep = configInfo.GetIntOrNull("FrameStep") ?? 0;
// must calculate this because used later on.
double frameOverlap = (frameSize - frameStep) / (double)frameSize;
// Default noiseReductionType = Standard
var bgNoiseThreshold = configInfo.GetDoubleOrNull("BgNoiseThreshold") ?? 3.0;
// EXTRACT ENVELOPE and SPECTROGRAM FROM RECORDING SEGMENT
var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recordingSegment, frameSize, frameStep);
var sonoConfig = new SonogramConfig()
{
epsilon = recordingSegment.Epsilon,
SampleRate = sampleRate,
WindowSize = frameSize,
WindowStep = frameStep,
WindowOverlap = frameOverlap,
WindowPower = dspOutput1.WindowPower,
Duration = recordingSegment.Duration,
NoiseReductionType = NoiseReductionType.Standard,
NoiseReductionParameter = bgNoiseThreshold,
};
// IMAGE 1) draw the WAVEFORM
if (doWaveForm)
{
var minValues = dspOutput1.MinFrameValues;
var maxValues = dspOutput1.MaxFrameValues;
int height = configInfo.GetIntOrNull("WaveformHeight") ?? 180;
var waveformImage = GetWaveformImage(minValues, maxValues, height);
// add in the title bar and time scales.
string title = $"WAVEFORM - {sourceRecordingName} (min value={dspOutput1.MinSignalValue:f3}, max value={dspOutput1.MaxSignalValue:f3})";
var titleBar = BaseSonogram.DrawTitleBarOfGrayScaleSpectrogram(title, waveformImage.Width);
var startTime = TimeSpan.Zero;
var xAxisTicInterval = TimeSpan.FromSeconds(1);
TimeSpan xAxisPixelDuration = TimeSpan.FromSeconds(frameStep / (double)sampleRate);
var labelInterval = TimeSpan.FromSeconds(5);
waveformImage = BaseSonogram.FrameSonogram(waveformImage, titleBar, startTime, xAxisTicInterval, xAxisPixelDuration, labelInterval);
list.Add(waveformImage);
}
// Draw various decibel spectrograms
if (doDecibelSpectrogram || doNoiseReducedSpectrogram || doDifferenceSpectrogram || doExperimentalSpectrogram)
{
// disable noise removal for first spectrogram
var disabledNoiseReductionType = sonoConfig.NoiseReductionType;
sonoConfig.NoiseReductionType = NoiseReductionType.None;
//Get the decibel spectrogram
var decibelSpectrogram = new SpectrogramStandard(sonoConfig, dspOutput1.AmplitudeSpectrogram);
result.DecibelSpectrogram = decibelSpectrogram;
double[,] dbSpectrogramData = (double[, ])decibelSpectrogram.Data.Clone();
// IMAGE 2) DecibelSpectrogram
if (doDecibelSpectrogram)
{
var image3 = decibelSpectrogram.GetImageFullyAnnotated($"DECIBEL SPECTROGRAM ({sourceRecordingName})");
list.Add(image3);
}
if (doNoiseReducedSpectrogram || doExperimentalSpectrogram || doDifferenceSpectrogram)
{
sonoConfig.NoiseReductionType = disabledNoiseReductionType;
sonoConfig.NoiseReductionParameter = bgNoiseThreshold;
double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram.Data);
decibelSpectrogram.Data = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram.Data, spectralDecibelBgn);
decibelSpectrogram.Data = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram.Data, nhThreshold: bgNoiseThreshold);
// IMAGE 3) DecibelSpectrogram - noise reduced
if (doNoiseReducedSpectrogram)
{
var image4 = decibelSpectrogram.GetImageFullyAnnotated($"DECIBEL SPECTROGRAM + Lamel noise subtraction. ({sourceRecordingName})");
list.Add(image4);
}
// IMAGE 4) EXPERIMENTAL Spectrogram
if (doExperimentalSpectrogram)
{
sonoConfig.NoiseReductionType = disabledNoiseReductionType;
var image5 = GetDecibelSpectrogram_Ridges(dbSpectrogramData, decibelSpectrogram, sourceRecordingName);
list.Add(image5);
}
// IMAGE 5) draw difference spectrogram
if (doDifferenceSpectrogram)
{
var differenceThreshold = configInfo.GetDoubleOrNull("DifferenceThreshold") ?? 3.0;
var image6 = GetDifferenceSpectrogram(dbSpectrogramData, differenceThreshold);
image6 = BaseSonogram.GetImageAnnotatedWithLinearHertzScale(image6, sampleRate, frameStep, $"DECIBEL DIFFERENCE SPECTROGRAM ({sourceRecordingName})");
list.Add(image6);
}
}
}
// IMAGE 6) Cepstral Spectrogram
if (doCepstralSpectrogram)
{
var image6 = GetCepstralSpectrogram(sonoConfig, recordingSegment, sourceRecordingName);
list.Add(image6);
}
// 7) AmplitudeSpectrogram_LocalContrastNormalization
if (doLcnSpectrogram)
{
var neighbourhoodSeconds = configInfo.GetDoubleOrNull("NeighbourhoodSeconds") ?? 0.5;
var lcnContrastParameter = configInfo.GetDoubleOrNull("LcnContrastLevel") ?? 0.4;
var image8 = GetLcnSpectrogram(sonoConfig, recordingSegment, sourceRecordingName, neighbourhoodSeconds, lcnContrastParameter);
list.Add(image8);
}
// 8) SOX SPECTROGRAM
//if (doSoxSpectrogram)
//{
//Log.Warn("SoX spectrogram set to true but is ignored when running as an IAnalyzer");
// The following parameters were once used to implement a sox spectrogram.
//bool makeSoxSonogram = configuration.GetBoolOrNull(AnalysisKeys.MakeSoxSonogram) ?? false;
//configDict[AnalysisKeys.SonogramTitle] = configuration[AnalysisKeys.SonogramTitle] ?? "Sonogram";
//configDict[AnalysisKeys.SonogramComment] = configuration[AnalysisKeys.SonogramComment] ?? "Sonogram produced using SOX";
//configDict[AnalysisKeys.SonogramColored] = configuration[AnalysisKeys.SonogramColored] ?? "false";
//configDict[AnalysisKeys.SonogramQuantisation] = configuration[AnalysisKeys.SonogramQuantisation] ?? "128";
//configDict[AnalysisKeys.AddTimeScale] = configuration[AnalysisKeys.AddTimeScale] ?? "true";
//configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
//configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
// var soxFile = new FileInfo(Path.Combine(output.FullName, sourceName + "SOX.png"));
// SpectrogramTools.MakeSonogramWithSox(sourceRecording, configDict, path2SoxSpectrogram);
// list.Add(image7);
//}
// COMBINE THE SPECTROGRAM IMAGES
result.CompositeImage = ImageTools.CombineImagesVertically(list);
return(result);
}
public static AudioToSonogramResult GenerateSpectrogramImages(FileInfo sourceRecording, Dictionary <string, string> configDict, DirectoryInfo outputDirectory)
{
// the source name was set up in the Analyse() method. But it could also be obtained directly from recording.
string sourceName = configDict[ConfigKeys.Recording.Key_RecordingFileName];
sourceName = Path.GetFileNameWithoutExtension(sourceName);
var result = new AudioToSonogramResult();
// init the image stack
var list = new List <Image>();
// 1) draw amplitude spectrogram
var recordingSegment = new AudioRecording(sourceRecording.FullName);
// default values config except disable noise removal for first two spectrograms
SonogramConfig sonoConfig = new SonogramConfig(configDict)
{
NoiseReductionType = NoiseReductionType.None
};
BaseSonogram sonogram = new AmplitudeSonogram(sonoConfig, recordingSegment.WavReader);
// remove the DC bin
sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.FrameCount - 1, sonogram.Configuration.FreqBinCount);
// save spectrogram data at this point - prior to noise reduction
double[,] spectrogramDataBeforeNoiseReduction = sonogram.Data;
const int lowPercentile = 20;
const double neighbourhoodSeconds = 0.25;
int neighbourhoodFrames = (int)(sonogram.FramesPerSecond * neighbourhoodSeconds);
const double lcnContrastLevel = 0.25;
////LoggedConsole.WriteLine("LCN: FramesPerSecond (Prior to LCN) = {0}", sonogram.FramesPerSecond);
////LoggedConsole.WriteLine("LCN: Neighbourhood of {0} seconds = {1} frames", neighbourhoodSeconds, neighbourhoodFrames);
sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_ShortRecordings_SubtractAndLCN(sonogram.Data, lowPercentile, neighbourhoodFrames, lcnContrastLevel);
// draw amplitude spectrogram unannotated
FileInfo outputImage1 = new FileInfo(Path.Combine(outputDirectory.FullName, sourceName + ".amplitd.bmp"));
ImageTools.DrawReversedMatrix(MatrixTools.MatrixRotate90Anticlockwise(sonogram.Data), outputImage1.FullName);
// draw amplitude spectrogram annotated
var image = sonogram.GetImageFullyAnnotated("AMPLITUDE SPECTROGRAM + Bin LCN (Local Contrast Normalisation)");
list.Add(image);
////string path2 = @"C:\SensorNetworks\Output\Sonograms\dataInput2.png";
////Histogram.DrawDistributionsAndSaveImage(sonogram.Data, path2);
// 2) A FALSE-COLOUR VERSION OF AMPLITUDE SPECTROGRAM
double ridgeThreshold = 0.20;
double[,] matrix = ImageTools.WienerFilter(sonogram.Data, 3);
byte[,] hits = RidgeDetection.Sobel5X5RidgeDetectionExperiment(matrix, ridgeThreshold);
hits = RidgeDetection.JoinDisconnectedRidgesInMatrix(hits, matrix, ridgeThreshold);
image = SpectrogramTools.CreateFalseColourAmplitudeSpectrogram(spectrogramDataBeforeNoiseReduction, null, hits);
image = sonogram.GetImageAnnotatedWithLinearHerzScale(image, "AMPLITUDE SPECTROGRAM + LCN + ridge detection");
list.Add(image);
Image envelopeImage = ImageTrack.DrawWaveEnvelopeTrack(recordingSegment, image.Width);
list.Add(envelopeImage);
// 3) now draw the standard decibel spectrogram
sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
// remove the DC bin
sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.FrameCount - 1, sonogram.Configuration.FreqBinCount);
// draw decibel spectrogram unannotated
FileInfo outputImage2 = new FileInfo(Path.Combine(outputDirectory.FullName, sourceName + ".deciBel.bmp"));
ImageTools.DrawReversedMatrix(MatrixTools.MatrixRotate90Anticlockwise(sonogram.Data), outputImage2.FullName);
image = sonogram.GetImageFullyAnnotated("DECIBEL SPECTROGRAM");
list.Add(image);
Image segmentationImage = ImageTrack.DrawSegmentationTrack(
sonogram,
EndpointDetectionConfiguration.K1Threshold,
EndpointDetectionConfiguration.K2Threshold,
image.Width);
list.Add(segmentationImage);
// keep the sonogram data (NOT noise reduced) for later use
double[,] dbSpectrogramData = (double[, ])sonogram.Data.Clone();
// 4) now draw the noise reduced decibel spectrogram
sonoConfig.NoiseReductionType = NoiseReductionType.Standard;
sonoConfig.NoiseReductionParameter = 3;
////sonoConfig.NoiseReductionType = NoiseReductionType.SHORT_RECORDING;
////sonoConfig.NoiseReductionParameter = 50;
sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
// draw decibel spectrogram unannotated
FileInfo outputImage3 = new FileInfo(Path.Combine(outputDirectory.FullName, sourceName + ".noNoise_dB.bmp"));
ImageTools.DrawReversedMatrix(MatrixTools.MatrixRotate90Anticlockwise(sonogram.Data), outputImage3.FullName);
image = sonogram.GetImageFullyAnnotated("DECIBEL SPECTROGRAM + Lamel noise subtraction");
list.Add(image);
// keep the sonogram data for later use
double[,] nrSpectrogramData = sonogram.Data;
// 5) A FALSE-COLOUR VERSION OF DECIBEL SPECTROGRAM
ridgeThreshold = 2.5;
matrix = ImageTools.WienerFilter(dbSpectrogramData, 3);
hits = RidgeDetection.Sobel5X5RidgeDetectionExperiment(matrix, ridgeThreshold);
image = SpectrogramTools.CreateFalseColourDecibelSpectrogram(dbSpectrogramData, nrSpectrogramData, hits);
image = sonogram.GetImageAnnotatedWithLinearHerzScale(image, "DECIBEL SPECTROGRAM - Colour annotated");
list.Add(image);
// 6) COMBINE THE SPECTROGRAM IMAGES
Image compositeImage = ImageTools.CombineImagesVertically(list);
FileInfo outputImage = new FileInfo(Path.Combine(outputDirectory.FullName, sourceName + ".5spectro.png"));
compositeImage.Save(outputImage.FullName, ImageFormat.Png);
result.SpectrogramFile = outputImage;
// 7) Generate the FREQUENCY x OSCILLATIONS Graphs and csv data
////bool saveData = true;
////bool saveImage = true;
////double[] oscillationsSpectrum = Oscillations2014.GenerateOscillationDataAndImages(sourceRecording, configDict, saveData, saveImage);
return(result);
}
public void TestFeatureLearning()
{
// var outputDir = this.outputDirectory;
var resultDir = PathHelper.ResolveAssetPath("FeatureLearning");
var folderPath = Path.Combine(resultDir, "random_audio_segments"); // Liz
// PathHelper.ResolveAssetPath(@"C:\Users\kholghim\Mahnoosh\PcaWhitening\random_audio_segments\1192_1000");
// var resultDir = PathHelper.ResolveAssetPath(@"C:\Users\kholghim\Mahnoosh\PcaWhitening");
var outputMelImagePath = Path.Combine(resultDir, "MelScaleSpectrogram.png");
var outputNormMelImagePath = Path.Combine(resultDir, "NormalizedMelScaleSpectrogram.png");
var outputNoiseReducedMelImagePath = Path.Combine(resultDir, "NoiseReducedMelSpectrogram.png");
var outputReSpecImagePath = Path.Combine(resultDir, "ReconstrcutedSpectrogram.png");
// var outputClusterImagePath = Path.Combine(resultDir, "Clusters.bmp");
// +++++++++++++++++++++++++++++++++++++++++++++++++patch sampling from 1000 random 1-min recordings from Gympie
// check whether there is any file in the folder/subfolders
if (Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories).Length == 0)
{
throw new ArgumentException("The folder of recordings is empty...");
}
// get the nyquist value from the first wav file in the folder of recordings
int nq = new AudioRecording(Directory.GetFiles(folderPath, "*.wav")[0]).Nyquist;
int nyquist = nq; // 11025;
int frameSize = 1024;
int finalBinCount = 128; // 256; // 100; // 40; // 200; //
int hertzInterval = 1000;
FreqScaleType scaleType = FreqScaleType.Mel;
var freqScale = new FrequencyScale(scaleType, nyquist, frameSize, finalBinCount, hertzInterval);
var fst = freqScale.ScaleType;
var sonoConfig = new SonogramConfig
{
WindowSize = frameSize,
// since each 24 frames duration is equal to 1 second
WindowOverlap = 0.1028,
DoMelScale = (scaleType == FreqScaleType.Mel) ? true : false,
MelBinCount = (scaleType == FreqScaleType.Mel) ? finalBinCount : frameSize / 2,
NoiseReductionType = NoiseReductionType.None,
};
/*
* // testing
* var recordingPath3 = PathHelper.ResolveAsset(folderPath, "SM304264_0+1_20160421_024539_46-47min.wav");
* var recording3 = new AudioRecording(recordingPath3);
* var sonogram3 = new SpectrogramStandard(sonoConfig, recording3.WavReader);
*
* // DO DRAW SPECTROGRAM
* var image4 = sonogram3.GetImageFullyAnnotated(sonogram3.GetImage(), "MELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
* image4.Save(outputMelImagePath);
*
* // Do RMS normalization
* sonogram3.Data = SNR.RmsNormalization(sonogram3.Data);
* var image5 = sonogram3.GetImageFullyAnnotated(sonogram3.GetImage(), "NORMALISEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
* image5.Save(outputNormMelImagePath);
*
* // NOISE REDUCTION
* sonogram3.Data = PcaWhitening.NoiseReduction(sonogram3.Data);
* var image6 = sonogram3.GetImageFullyAnnotated(sonogram3.GetImage(), "NOISEREDUCEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
* image6.Save(outputNoiseReducedMelImagePath);
*
* //testing
*/
// Define the minFreBin and MaxFreqBin to be able to work at arbitrary frequency bin bounds.
// The default value is minFreqBin = 1 and maxFreqBin = finalBinCount.
// To work with arbitrary frequency bin bounds we need to manually set these two parameters.
int minFreqBin = 40; //1
int maxFreqBin = 80; //finalBinCount;
int numFreqBand = 1; //4;
int patchWidth = (maxFreqBin - minFreqBin + 1) / numFreqBand; // finalBinCount / numFreqBand;
int patchHeight = 1; // 2; // 4; // 16; // 6; // Frame size
int numRandomPatches = 20; // 40; // 80; // 30; // 100; // 500; //
// int fileCount = Directory.GetFiles(folderPath, "*.wav").Length;
// Define variable number of "randomPatch" lists based on "numFreqBand"
Dictionary <string, List <double[, ]> > randomPatchLists = new Dictionary <string, List <double[, ]> >();
for (int i = 0; i < numFreqBand; i++)
{
randomPatchLists.Add(string.Format("randomPatch{0}", i.ToString()), new List <double[, ]>());
}
List <double[, ]> randomPatches = new List <double[, ]>();
/*
* foreach (string filePath in Directory.GetFiles(folderPath, "*.wav"))
* {
* FileInfo f = filePath.ToFileInfo();
* if (f.Length == 0)
* {
* Debug.WriteLine(f.Name);
* }
* }
*/
double[,] inputMatrix;
foreach (string filePath in Directory.GetFiles(folderPath, "*.wav"))
{
FileInfo fileInfo = filePath.ToFileInfo();
// process the wav file if it is not empty
if (fileInfo.Length != 0)
{
var recording = new AudioRecording(filePath);
sonoConfig.SourceFName = recording.BaseName;
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// DO RMS NORMALIZATION
sonogram.Data = SNR.RmsNormalization(sonogram.Data);
// DO NOISE REDUCTION
// sonogram.Data = SNR.NoiseReduce_Median(sonogram.Data, nhBackgroundThreshold: 2.0);
sonogram.Data = PcaWhitening.NoiseReduction(sonogram.Data);
// check whether the full band spectrogram is needed or a matrix with arbitrary freq bins
if (minFreqBin != 1 || maxFreqBin != finalBinCount)
{
inputMatrix = PatchSampling.GetArbitraryFreqBandMatrix(sonogram.Data, minFreqBin, maxFreqBin);
}
else
{
inputMatrix = sonogram.Data;
}
// creating matrices from different freq bands of the source spectrogram
List <double[, ]> allSubmatrices = PatchSampling.GetFreqBandMatrices(inputMatrix, numFreqBand);
// Second: selecting random patches from each freq band matrix and add them to the corresponding patch list
int count = 0;
while (count < allSubmatrices.Count)
{
randomPatchLists[$"randomPatch{count.ToString()}"].Add(PatchSampling
.GetPatches(allSubmatrices.ToArray()[count], patchWidth, patchHeight, numRandomPatches,
PatchSampling.SamplingMethod.Random).ToMatrix());
count++;
}
}
}
foreach (string key in randomPatchLists.Keys)
{
randomPatches.Add(PatchSampling.ListOf2DArrayToOne2DArray(randomPatchLists[key]));
}
// convert list of random patches matrices to one matrix
int numberOfClusters = 50; //256; // 128; // 64; // 32; // 10; //
List <double[][]> allBandsCentroids = new List <double[][]>();
List <KMeansClusterCollection> allClusteringOutput = new List <KMeansClusterCollection>();
for (int i = 0; i < randomPatches.Count; i++)
{
double[,] patchMatrix = randomPatches[i];
// Apply PCA Whitening
var whitenedSpectrogram = PcaWhitening.Whitening(true, patchMatrix);
// Do k-means clustering
var clusteringOutput = KmeansClustering.Clustering(whitenedSpectrogram.Reversion, numberOfClusters);
// var clusteringOutput = KmeansClustering.Clustering(patchMatrix, noOfClusters, pathToClusterCsvFile);
// writing centroids to a csv file
// note that Csv.WriteToCsv can't write data types like dictionary<int, double[]> (problems with arrays)
// I converted the dictionary values to a matrix and used the Csv.WriteMatrixToCsv
// it might be a better way to do this
string pathToClusterCsvFile = Path.Combine(resultDir, "ClusterCentroids" + i.ToString() + ".csv");
var clusterCentroids = clusteringOutput.ClusterIdCentroid.Values.ToArray();
Csv.WriteMatrixToCsv(pathToClusterCsvFile.ToFileInfo(), clusterCentroids.ToMatrix());
//Csv.WriteToCsv(pathToClusterCsvFile.ToFileInfo(), clusterCentroids);
// sorting clusters based on size and output it to a csv file
Dictionary <int, double> clusterIdSize = clusteringOutput.ClusterIdSize;
int[] sortOrder = KmeansClustering.SortClustersBasedOnSize(clusterIdSize);
// Write cluster ID and size to a CSV file
string pathToClusterSizeCsvFile = Path.Combine(resultDir, "ClusterSize" + i.ToString() + ".csv");
Csv.WriteToCsv(pathToClusterSizeCsvFile.ToFileInfo(), clusterIdSize);
// Draw cluster image directly from clustering output
List <KeyValuePair <int, double[]> > list = clusteringOutput.ClusterIdCentroid.ToList();
double[][] centroids = new double[list.Count][];
for (int j = 0; j < list.Count; j++)
{
centroids[j] = list[j].Value;
}
allBandsCentroids.Add(centroids);
allClusteringOutput.Add(clusteringOutput.Clusters);
List <double[, ]> allCentroids = new List <double[, ]>();
for (int k = 0; k < centroids.Length; k++)
{
// convert each centroid to a matrix in order of cluster ID
// double[,] cent = PatchSampling.ArrayToMatrixByColumn(centroids[i], patchWidth, patchHeight);
// OR: in order of cluster size
double[,] cent =
MatrixTools.ArrayToMatrixByColumn(centroids[sortOrder[k]], patchWidth, patchHeight);
// normalize each centroid
double[,] normCent = DataTools.normalise(cent);
// add a row of zero to each centroid
double[,] cent2 = PatchSampling.AddRow(normCent);
allCentroids.Add(cent2);
}
// concatenate all centroids
double[,] mergedCentroidMatrix = PatchSampling.ListOf2DArrayToOne2DArray(allCentroids);
// Draw clusters
// int gridInterval = 1000;
// var freqScale = new FrequencyScale(FreqScaleType.Mel, nyquist, frameSize, finalBinCount, gridInterval);
var clusterImage = ImageTools.DrawMatrixWithoutNormalisation(mergedCentroidMatrix);
clusterImage.RotateFlip(RotateFlipType.Rotate270FlipNone);
// clusterImage.Save(outputClusterImagePath, ImageFormat.Bmp);
var outputClusteringImage = Path.Combine(resultDir, "ClustersWithGrid" + i.ToString() + ".bmp");
// Image bmp = Image.Load<Rgb24>(filename);
FrequencyScale.DrawFrequencyLinesOnImage((Image <Rgb24>)clusterImage, freqScale, includeLabels: false);
clusterImage.Save(outputClusteringImage);
}
//+++++++++++++++++++++++++++++++++++++++++++++++++++++Processing and generating features for the target recordings
var recording2Path = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav");
// var recording2Path = PathHelper.ResolveAsset(folderPath, "gympie_np_1192_353972_20160303_055854_60_0.wav"); // folder with 1000 files
// var recording2Path = PathHelper.ResolveAsset(folderPath, "gympie_np_1192_353887_20151230_042625_60_0.wav"); // folder with 1000 files
// var recording2Path = PathHelper.ResolveAsset(folderPath, "gympie_np_1192_354744_20151018_053923_60_0.wav"); // folder with 100 files
var recording2 = new AudioRecording(recording2Path);
var sonogram2 = new SpectrogramStandard(sonoConfig, recording2.WavReader);
// DO DRAW SPECTROGRAM
var image = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "MELSPECTROGRAM: " + fst.ToString(),
freqScale.GridLineLocations);
image.Save(outputMelImagePath);
// Do RMS normalization
sonogram2.Data = SNR.RmsNormalization(sonogram2.Data);
var image2 = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(),
"NORMALISEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
image2.Save(outputNormMelImagePath);
// NOISE REDUCTION
sonogram2.Data = PcaWhitening.NoiseReduction(sonogram2.Data);
var image3 = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(),
"NOISEREDUCEDMELSPECTROGRAM: " + fst.ToString(), freqScale.GridLineLocations);
image3.Save(outputNoiseReducedMelImagePath);
// check whether the full band spectrogram is needed or a matrix with arbitrary freq bins
if (minFreqBin != 1 || maxFreqBin != finalBinCount)
{
inputMatrix = PatchSampling.GetArbitraryFreqBandMatrix(sonogram2.Data, minFreqBin, maxFreqBin);
}
else
{
inputMatrix = sonogram2.Data;
}
// extracting sequential patches from the target spectrogram
List <double[, ]> allSubmatrices2 = PatchSampling.GetFreqBandMatrices(inputMatrix, numFreqBand);
double[][,] matrices2 = allSubmatrices2.ToArray();
List <double[, ]> allSequentialPatchMatrix = new List <double[, ]>();
for (int i = 0; i < matrices2.GetLength(0); i++)
{
int rows = matrices2[i].GetLength(0);
int columns = matrices2[i].GetLength(1);
var sequentialPatches = PatchSampling.GetPatches(matrices2[i], patchWidth, patchHeight,
(rows / patchHeight) * (columns / patchWidth), PatchSampling.SamplingMethod.Sequential);
allSequentialPatchMatrix.Add(sequentialPatches.ToMatrix());
}
// +++++++++++++++++++++++++++++++++++Feature Transformation
// to do the feature transformation, we normalize centroids and
// sequential patches from the input spectrogram to unit length
// Then, we calculate the dot product of each patch with the centroids' matrix
List <double[][]> allNormCentroids = new List <double[][]>();
for (int i = 0; i < allBandsCentroids.Count; i++)
{
// double check the index of the list
double[][] normCentroids = new double[allBandsCentroids.ToArray()[i].GetLength(0)][];
for (int j = 0; j < allBandsCentroids.ToArray()[i].GetLength(0); j++)
{
normCentroids[j] = ART_2A.NormaliseVector(allBandsCentroids.ToArray()[i][j]);
}
allNormCentroids.Add(normCentroids);
}
List <double[][]> allFeatureTransVectors = new List <double[][]>();
for (int i = 0; i < allSequentialPatchMatrix.Count; i++)
{
double[][] featureTransVectors = new double[allSequentialPatchMatrix.ToArray()[i].GetLength(0)][];
for (int j = 0; j < allSequentialPatchMatrix.ToArray()[i].GetLength(0); j++)
{
var normVector =
ART_2A.NormaliseVector(allSequentialPatchMatrix.ToArray()[i]
.ToJagged()[j]); // normalize each patch to unit length
featureTransVectors[j] = allNormCentroids.ToArray()[i].ToMatrix().Dot(normVector);
}
allFeatureTransVectors.Add(featureTransVectors);
}
// +++++++++++++++++++++++++++++++++++Feature Transformation
// +++++++++++++++++++++++++++++++++++Temporal Summarization
// The resolution to generate features is 1 second
// Each 24 single-frame patches form 1 second
// for each 24 patch, we generate 3 vectors of mean, std, and max
// The pre-assumption is that each input spectrogram is 1 minute
List <double[, ]> allMeanFeatureVectors = new List <double[, ]>();
List <double[, ]> allMaxFeatureVectors = new List <double[, ]>();
List <double[, ]> allStdFeatureVectors = new List <double[, ]>();
// number of frames needs to be concatenated to form 1 second. Each 24 frames make 1 second.
int numFrames = (24 / patchHeight) * 60;
foreach (var freqBandFeature in allFeatureTransVectors)
{
// store features of different bands in lists
List <double[]> meanFeatureVectors = new List <double[]>();
List <double[]> maxFeatureVectors = new List <double[]>();
List <double[]> stdFeatureVectors = new List <double[]>();
int c = 0;
while (c + numFrames < freqBandFeature.GetLength(0))
{
// First, make a list of patches that would be equal to 1 second
List <double[]> sequencesOfFramesList = new List <double[]>();
for (int i = c; i < c + numFrames; i++)
{
sequencesOfFramesList.Add(freqBandFeature[i]);
}
List <double> mean = new List <double>();
List <double> std = new List <double>();
List <double> max = new List <double>();
double[,] sequencesOfFrames = sequencesOfFramesList.ToArray().ToMatrix();
// int len = sequencesOfFrames.GetLength(1);
// Second, calculate mean, max, and standard deviation of six vectors element-wise
for (int j = 0; j < sequencesOfFrames.GetLength(1); j++)
{
double[] temp = new double[sequencesOfFrames.GetLength(0)];
for (int k = 0; k < sequencesOfFrames.GetLength(0); k++)
{
temp[k] = sequencesOfFrames[k, j];
}
mean.Add(AutoAndCrossCorrelation.GetAverage(temp));
std.Add(AutoAndCrossCorrelation.GetStdev(temp));
max.Add(temp.GetMaxValue());
}
meanFeatureVectors.Add(mean.ToArray());
maxFeatureVectors.Add(max.ToArray());
stdFeatureVectors.Add(std.ToArray());
c += numFrames;
}
allMeanFeatureVectors.Add(meanFeatureVectors.ToArray().ToMatrix());
allMaxFeatureVectors.Add(maxFeatureVectors.ToArray().ToMatrix());
allStdFeatureVectors.Add(stdFeatureVectors.ToArray().ToMatrix());
}
// +++++++++++++++++++++++++++++++++++Temporal Summarization
// ++++++++++++++++++++++++++++++++++Writing features to file
// First, concatenate mean, max, std for each second.
// Then write to CSV file.
for (int j = 0; j < allMeanFeatureVectors.Count; j++)
{
// write the features of each pre-defined frequency band into a separate CSV file
var outputFeatureFile = Path.Combine(resultDir, "FeatureVectors" + j.ToString() + ".csv");
// creating the header for CSV file
List <string> header = new List <string>();
for (int i = 0; i < allMeanFeatureVectors.ToArray()[j].GetLength(1); i++)
{
header.Add("mean" + i.ToString());
}
for (int i = 0; i < allMaxFeatureVectors.ToArray()[j].GetLength(1); i++)
{
header.Add("max" + i.ToString());
}
for (int i = 0; i < allStdFeatureVectors.ToArray()[j].GetLength(1); i++)
{
header.Add("std" + i.ToString());
}
// concatenating mean, std, and max vector together for each 1 second
List <double[]> featureVectors = new List <double[]>();
for (int i = 0; i < allMeanFeatureVectors.ToArray()[j].ToJagged().GetLength(0); i++)
{
List <double[]> featureList = new List <double[]>
{
allMeanFeatureVectors.ToArray()[j].ToJagged()[i],
allMaxFeatureVectors.ToArray()[j].ToJagged()[i],
allStdFeatureVectors.ToArray()[j].ToJagged()[i],
};
double[] featureVector = DataTools.ConcatenateVectors(featureList);
featureVectors.Add(featureVector);
}
// writing feature vectors to CSV file
using (StreamWriter file = new StreamWriter(outputFeatureFile))
{
// writing the header to CSV file
foreach (var entry in header.ToArray())
{
file.Write(entry + ",");
}
file.Write(Environment.NewLine);
foreach (var entry in featureVectors.ToArray())
{
foreach (var value in entry)
{
file.Write(value + ",");
}
file.Write(Environment.NewLine);
}
}
}
/*
* // Reconstructing the target spectrogram based on clusters' centroids
* List<double[,]> convertedSpec = new List<double[,]>();
* int columnPerFreqBand = sonogram2.Data.GetLength(1) / numFreqBand;
* for (int i = 0; i < allSequentialPatchMatrix.Count; i++)
* {
* double[,] reconstructedSpec2 = KmeansClustering.ReconstructSpectrogram(allSequentialPatchMatrix.ToArray()[i], allClusteringOutput.ToArray()[i]);
* convertedSpec.Add(PatchSampling.ConvertPatches(reconstructedSpec2, patchWidth, patchHeight, columnPerFreqBand));
* }
*
* sonogram2.Data = PatchSampling.ConcatFreqBandMatrices(convertedSpec);
*
* // DO DRAW SPECTROGRAM
* var reconstructedSpecImage = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "RECONSTRUCTEDSPECTROGRAM: " + freqScale.ScaleType.ToString(), freqScale.GridLineLocations);
* reconstructedSpecImage.Save(outputReSpecImagePath);
*/
}
public void TestKmeansClustering()
{
var outputDir = this.outputDirectory;
var recordingsPath = PathHelper.ResolveAssetPath("FeatureLearning");
var folderPath = Path.Combine(recordingsPath, "random_audio_segments");
var outputImagePath = Path.Combine(outputDir.FullName, "ReconstrcutedSpectrogram.png");
// check whether there is any file in the folder/subfolders
if (Directory.GetFiles(folderPath, "*", SearchOption.AllDirectories).Length == 0)
{
throw new ArgumentException("The folder of recordings is empty. Test will fail!");
}
// get the nyquist value from the first wav file in the folder of recordings
int nq = new AudioRecording(Directory.GetFiles(folderPath, "*.wav")[0]).Nyquist;
int nyquist = nq;
int frameSize = 1024;
int finalBinCount = 128;
int hertzInterval = 1000;
FreqScaleType scaleType = FreqScaleType.Mel;
var freqScale = new FrequencyScale(scaleType, nyquist, frameSize, finalBinCount, hertzInterval);
var sonoConfig = new SonogramConfig
{
WindowSize = frameSize,
//WindowOverlap is set based on the fact that each 24 frames is equal to 1 second
WindowOverlap = 0.1028,
DoMelScale = (scaleType == FreqScaleType.Mel) ? true : false,
MelBinCount = (scaleType == FreqScaleType.Mel) ? finalBinCount : frameSize / 2,
NoiseReductionType = NoiseReductionType.None,
};
int numberOfFreqBand = 4;
int patchWidth = finalBinCount / numberOfFreqBand;
int patchHeight = 1;
int numberOfRandomPatches = 20;
// Define variable number of "randomPatch" lists based on "numberOfFreqBand"
Dictionary <string, List <double[, ]> > randomPatchLists = new Dictionary <string, List <double[, ]> >();
for (int i = 0; i < numberOfFreqBand; i++)
{
randomPatchLists.Add(string.Format("randomPatch{0}", i.ToString()), new List <double[, ]>());
}
List <double[, ]> randomPatches = new List <double[, ]>();
foreach (string filePath in Directory.GetFiles(folderPath, "*.wav"))
{
FileInfo fileInfo = filePath.ToFileInfo();
// process the wav file if it is not empty
if (fileInfo.Length != 0)
{
var recording = new AudioRecording(filePath);
sonoConfig.SourceFName = recording.BaseName;
var sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
// DO RMS NORMALIZATION
sonogram.Data = SNR.RmsNormalization(sonogram.Data);
// DO NOISE REDUCTION
sonogram.Data = PcaWhitening.NoiseReduction(sonogram.Data);
// creating matrices from different freq bands of the source spectrogram
List <double[, ]> allSubmatrices = PatchSampling.GetFreqBandMatrices(sonogram.Data, numberOfFreqBand);
// Second: selecting random patches from each freq band matrix and add them to the corresponding patch list
int count = 0;
while (count < allSubmatrices.Count)
{
randomPatchLists[string.Format("randomPatch{0}", count.ToString())].Add(PatchSampling.GetPatches(allSubmatrices.ToArray()[count], patchWidth, patchHeight, numberOfRandomPatches, PatchSampling.SamplingMethod.Random).ToMatrix());
count++;
}
}
}
foreach (string key in randomPatchLists.Keys)
{
randomPatches.Add(PatchSampling.ListOf2DArrayToOne2DArray(randomPatchLists[key]));
}
// convert list of random patches matrices to one matrix
int numberOfClusters = 32;
List <KMeansClusterCollection> allClusteringOutput = new List <KMeansClusterCollection>();
for (int i = 0; i < randomPatches.Count; i++)
{
double[,] patchMatrix = randomPatches[i];
// Do k-means clustering
string pathToClusterCsvFile = Path.Combine(outputDir.FullName, "ClusterCentroids" + i.ToString() + ".csv");
var clusteringOutput = KmeansClustering.Clustering(patchMatrix, numberOfClusters);
// sorting clusters based on size and output it to a csv file
Dictionary <int, double> clusterIdSize = clusteringOutput.ClusterIdSize;
int[] sortOrder = KmeansClustering.SortClustersBasedOnSize(clusterIdSize);
// Write cluster ID and size to a CSV file
string pathToClusterSizeCsvFile = Path.Combine(outputDir.FullName, "ClusterSize" + i.ToString() + ".csv");
Csv.WriteToCsv(pathToClusterSizeCsvFile.ToFileInfo(), clusterIdSize);
// Draw cluster image directly from clustering output
List <KeyValuePair <int, double[]> > listCluster = clusteringOutput.ClusterIdCentroid.ToList();
double[][] centroids = new double[listCluster.Count][];
for (int j = 0; j < listCluster.Count; j++)
{
centroids[j] = listCluster[j].Value;
}
allClusteringOutput.Add(clusteringOutput.Clusters);
List <double[, ]> allCentroids = new List <double[, ]>();
for (int k = 0; k < centroids.Length; k++)
{
// convert each centroid to a matrix in order of cluster ID
// OR: in order of cluster size
double[,] centroid = MatrixTools.ArrayToMatrixByColumn(centroids[sortOrder[k]], patchWidth, patchHeight);
// normalize each centroid
double[,] normalizedCentroid = DataTools.normalise(centroid);
// add a row of zero to each centroid
double[,] newCentroid = PatchSampling.AddRow(normalizedCentroid);
allCentroids.Add(newCentroid);
}
// concatenate all centroids
double[,] mergedCentroidMatrix = PatchSampling.ListOf2DArrayToOne2DArray(allCentroids);
// Draw clusters
var clusterImage = ImageTools.DrawMatrixWithoutNormalisation(mergedCentroidMatrix);
clusterImage.RotateFlip(RotateFlipType.Rotate270FlipNone);
var outputClusteringImage = Path.Combine(outputDir.FullName, "ClustersWithGrid" + i.ToString() + ".bmp");
FrequencyScale.DrawFrequencyLinesOnImage((Bitmap)clusterImage, freqScale, includeLabels: false);
clusterImage.Save(outputClusteringImage);
}
//+++++++++++++++++++++++++++++++++++++++++++Reconstructing a target spectrogram from sequential patches and the cluster centroids
var recording2Path = PathHelper.ResolveAsset("Recordings", "BAC2_20071008-085040.wav");
var recording2 = new AudioRecording(recording2Path);
var sonogram2 = new SpectrogramStandard(sonoConfig, recording2.WavReader);
var targetSpec = sonogram2.Data;
// Do RMS normalization
sonogram2.Data = SNR.RmsNormalization(sonogram2.Data);
// NOISE REDUCTION
sonogram2.Data = PcaWhitening.NoiseReduction(sonogram2.Data);
// extracting sequential patches from the target spectrogram
List <double[, ]> allSubmatrices2 = PatchSampling.GetFreqBandMatrices(sonogram2.Data, numberOfFreqBand);
double[][,] matrices2 = allSubmatrices2.ToArray();
List <double[, ]> allSequentialPatchMatrix = new List <double[, ]>();
for (int i = 0; i < matrices2.GetLength(0); i++)
{
int rows = matrices2[i].GetLength(0);
int columns = matrices2[i].GetLength(1);
var sequentialPatches = PatchSampling.GetPatches(matrices2[i], patchWidth, patchHeight, (rows / patchHeight) * (columns / patchWidth), PatchSampling.SamplingMethod.Sequential);
allSequentialPatchMatrix.Add(sequentialPatches.ToMatrix());
}
List <double[, ]> convertedSpectrogram = new List <double[, ]>();
int columnPerFreqBand = sonogram2.Data.GetLength(1) / numberOfFreqBand;
for (int i = 0; i < allSequentialPatchMatrix.Count; i++)
{
double[,] reconstructedSpec2 = KmeansClustering.ReconstructSpectrogram(allSequentialPatchMatrix.ToArray()[i], allClusteringOutput.ToArray()[i]);
convertedSpectrogram.Add(PatchSampling.ConvertPatches(reconstructedSpec2, patchWidth, patchHeight, columnPerFreqBand));
}
sonogram2.Data = PatchSampling.ConcatFreqBandMatrices(convertedSpectrogram);
// DO DRAW SPECTROGRAM
var reconstructedSpecImage = sonogram2.GetImageFullyAnnotated(sonogram2.GetImage(), "RECONSTRUCTEDSPECTROGRAM: " + freqScale.ScaleType.ToString(), freqScale.GridLineLocations);
reconstructedSpecImage.Save(outputImagePath, ImageFormat.Png);
// DO UNIT TESTING
Assert.AreEqual(targetSpec.GetLength(0), sonogram2.Data.GetLength(0));
Assert.AreEqual(targetSpec.GetLength(1), sonogram2.Data.GetLength(1));
}
public static void Main(Arguments arguments)
{
// 1. set up the necessary files
FileInfo sourceRecording = arguments.Source;
FileInfo configFile = arguments.Config.ToFileInfo();
DirectoryInfo opDir = arguments.Output;
opDir.Create();
if (arguments.StartOffset.HasValue ^ arguments.EndOffset.HasValue)
{
throw new InvalidStartOrEndException("If StartOffset or EndOffset is specified, then both must be specified");
}
var offsetsProvided = arguments.StartOffset.HasValue && arguments.EndOffset.HasValue;
// set default offsets - only use defaults if not provided in argments list
TimeSpan?startOffset = null;
TimeSpan?endOffset = null;
if (offsetsProvided)
{
startOffset = TimeSpan.FromSeconds(arguments.StartOffset.Value);
endOffset = TimeSpan.FromSeconds(arguments.EndOffset.Value);
}
const string Title = "# MAKE A SONOGRAM FROM AUDIO RECORDING and do OscillationsGeneric activity.";
string date = "# DATE AND TIME: " + DateTime.Now;
LoggedConsole.WriteLine(Title);
LoggedConsole.WriteLine(date);
LoggedConsole.WriteLine("# Input audio file: " + sourceRecording.Name);
string sourceName = Path.GetFileNameWithoutExtension(sourceRecording.FullName);
// 2. get the config dictionary
Config configuration = ConfigFile.Deserialize(configFile);
// below three lines are examples of retrieving info from Config config
// string analysisIdentifier = configuration[AnalysisKeys.AnalysisName];
// bool saveIntermediateWavFiles = (bool?)configuration[AnalysisKeys.SaveIntermediateWavFiles] ?? false;
// scoreThreshold = (double?)configuration[AnalysisKeys.EventThreshold] ?? scoreThreshold;
// Resample rate must be 2 X the desired Nyquist. Default is that of recording.
var resampleRate = configuration.GetIntOrNull(AnalysisKeys.ResampleRate) ?? AppConfigHelper.DefaultTargetSampleRate;
var configDict = new Dictionary <string, string>(configuration.ToDictionary());
// #NOISE REDUCTION PARAMETERS
//string noisereduce = configDict[ConfigKeys.Mfcc.Key_NoiseReductionType];
configDict[AnalysisKeys.NoiseDoReduction] = "false";
configDict[AnalysisKeys.NoiseReductionType] = "NONE";
configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
configDict[ConfigKeys.Recording.Key_RecordingCallName] = sourceRecording.FullName;
configDict[ConfigKeys.Recording.Key_RecordingFileName] = sourceRecording.Name;
configDict[AnalysisKeys.AddTimeScale] = configuration[AnalysisKeys.AddTimeScale] ?? "true";
configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
// ####################################################################
// print out the sonogram parameters
LoggedConsole.WriteLine("\nPARAMETERS");
foreach (KeyValuePair <string, string> kvp in configDict)
{
LoggedConsole.WriteLine("{0} = {1}", kvp.Key, kvp.Value);
}
LoggedConsole.WriteLine("Sample Length for detecting oscillations = {0}", SampleLength);
// 3: GET RECORDING
FileInfo tempAudioSegment = new FileInfo(Path.Combine(opDir.FullName, "tempWavFile.wav"));
// delete the temp audio file if it already exists.
if (File.Exists(tempAudioSegment.FullName))
{
File.Delete(tempAudioSegment.FullName);
}
// This line creates a temporary version of the source file downsampled as per entry in the config file
MasterAudioUtility.SegmentToWav(sourceRecording, tempAudioSegment, new AudioUtilityRequest()
{
TargetSampleRate = resampleRate
});
// 1) get amplitude spectrogram
AudioRecording recordingSegment = new AudioRecording(tempAudioSegment.FullName);
SonogramConfig sonoConfig = new SonogramConfig(configDict); // default values config
BaseSonogram sonogram = new AmplitudeSonogram(sonoConfig, recordingSegment.WavReader);
Console.WriteLine("FramesPerSecond = {0}", sonogram.FramesPerSecond);
// remove the DC bin
sonogram.Data = MatrixTools.Submatrix(sonogram.Data, 0, 1, sonogram.FrameCount - 1, sonogram.Configuration.FreqBinCount);
// ###############################################################
// DO LocalContrastNormalisation
//int fieldSize = 9;
//sonogram.Data = LocalContrastNormalisation.ComputeLCN(sonogram.Data, fieldSize);
// LocalContrastNormalisation over frequency bins is better and faster.
int neighbourhood = 15;
double contrastLevel = 0.5;
sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLCNDivision(sonogram.Data, neighbourhood, contrastLevel);
// ###############################################################
// lowering the sensitivity threshold increases the number of hits.
if (configDict.ContainsKey(AnalysisKeys.OscilDetection2014SensitivityThreshold))
{
Oscillations2014.DefaultSensitivityThreshold = double.Parse(configDict[AnalysisKeys.OscilDetection2014SensitivityThreshold]);
}
if (configDict.ContainsKey(AnalysisKeys.OscilDetection2014SampleLength))
{
Oscillations2014.DefaultSampleLength = int.Parse(configDict[AnalysisKeys.OscilDetection2014SensitivityThreshold]);
}
var list1 = new List <Image>();
//var result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, 64, "Autocorr-FFT");
//list1.Add(result.FreqOscillationImage);
var result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, "Autocorr-FFT");
list1.Add(result.FreqOscillationImage);
result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, "Autocorr-SVD-FFT");
list1.Add(result.FreqOscillationImage);
result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, "Autocorr-WPD");
list1.Add(result.FreqOscillationImage);
Image compositeOscImage1 = ImageTools.CombineImagesInLine(list1.ToArray());
// ###############################################################
// init the sonogram image stack
var sonogramList = new List <Image>();
var image = sonogram.GetImageFullyAnnotated("AMPLITUDE SPECTROGRAM");
sonogramList.Add(image);
//string testPath = @"C:\SensorNetworks\Output\Sonograms\amplitudeSonogram.png";
//image.Save(testPath, ImageFormat.Png);
Image envelopeImage = ImageTrack.DrawWaveEnvelopeTrack(recordingSegment, image.Width);
sonogramList.Add(envelopeImage);
// 2) now draw the standard decibel spectrogram
sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
// ###############################################################
list1 = new List <Image>();
//result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, 64, "Autocorr-FFT");
//list1.Add(result.FreqOscillationImage);
result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, "Autocorr-FFT");
list1.Add(result.FreqOscillationImage);
result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, "Autocorr-SVD-FFT");
list1.Add(result.FreqOscillationImage);
result = Oscillations2014.GetFreqVsOscillationsDataAndImage(sonogram, "Autocorr-WPD");
list1.Add(result.FreqOscillationImage);
Image compositeOscImage2 = ImageTools.CombineImagesInLine(list1.ToArray());
// ###############################################################
//image = sonogram.GetImageFullyAnnotated("DECIBEL SPECTROGRAM");
//list.Add(image);
// combine eight images
list1 = new List <Image>();
list1.Add(compositeOscImage1);
list1.Add(compositeOscImage2);
Image compositeOscImage3 = ImageTools.CombineImagesVertically(list1.ToArray());
string imagePath3 = Path.Combine(opDir.FullName, sourceName + "_freqOscilMatrix.png");
compositeOscImage3.Save(imagePath3, ImageFormat.Png);
Image segmentationImage = ImageTrack.DrawSegmentationTrack(
sonogram,
EndpointDetectionConfiguration.K1Threshold,
EndpointDetectionConfiguration.K2Threshold,
image.Width);
sonogramList.Add(segmentationImage);
// 3) now draw the noise reduced decibel spectrogram
sonoConfig.NoiseReductionType = NoiseReductionType.Standard;
sonoConfig.NoiseReductionParameter = configuration.GetDoubleOrNull(AnalysisKeys.NoiseBgThreshold) ?? 3.0;
sonogram = new SpectrogramStandard(sonoConfig, recordingSegment.WavReader);
image = sonogram.GetImageFullyAnnotated("NOISE-REDUCED DECIBEL SPECTROGRAM");
sonogramList.Add(image);
// ###############################################################
// deriving osscilation graph from this noise reduced spectrogram did not work well
//Oscillations2014.SaveFreqVsOscillationsDataAndImage(sonogram, sampleLength, algorithmName, opDir);
// ###############################################################
Image compositeSonogram = ImageTools.CombineImagesVertically(sonogramList);
string imagePath2 = Path.Combine(opDir.FullName, sourceName + ".png");
compositeSonogram.Save(imagePath2, ImageFormat.Png);
LoggedConsole.WriteLine("\n##### FINISHED FILE ###################################################\n");
}
