public void TestChromelessImage()
{
var indexPropertiesFile = ConfigFile.Default <IndexPropertiesCollection>();
var indexProperties = ConfigFile.Deserialize <IndexPropertiesCollection>(indexPropertiesFile);
var indexSpectrograms = new Dictionary <string, double[, ]>(6);
var indexStatistics = new Dictionary <string, IndexDistributions.SpectralStats>();
var keys = (LDSpectrogramRGB.DefaultColorMap1 + "-" + LDSpectrogramRGB.DefaultColorMap2).Split('-');
foreach (var key in keys)
{
var matrix = new double[256, 60].Fill(indexProperties[key].DefaultValue);
indexSpectrograms.Add(key, matrix);
double[] array = DataTools.Matrix2Array(matrix);
indexStatistics.Add(key, IndexDistributions.GetModeAndOneTailedStandardDeviation(array, 300, IndexDistributions.UpperPercentileDefault));
}
var images = LDSpectrogramRGB.DrawSpectrogramsFromSpectralIndices(
inputDirectory: null,
outputDirectory: this.outputDirectory,
ldSpectrogramConfig: new LdSpectrogramConfig(),
indexPropertiesConfigPath: indexPropertiesFile,
indexGenerationData: new IndexGenerationData()
{
AnalysisStartOffset = 0.Seconds(),
FrameLength = 512,
IndexCalculationDuration = 60.0.Seconds(),
RecordingBasename = "RGB_TEST",
RecordingDuration = 60.0.Seconds(),
SampleRateResampled = 22050,
},
basename: "RGB_TEST",
analysisType: AcousticIndices.AnalysisName,
indexSpectrograms: indexSpectrograms,
summaryIndices: Enumerable
.Range(0, 60)
.Select((x) => new SummaryIndexValues(60.0.Seconds(), indexProperties))
.Cast <SummaryIndexBase>()
.ToArray(),
indexStatistics: indexStatistics,
imageChrome: ImageChrome.Without);
foreach (var(image, key) in images)
{
Assert.That.ImageIsSize(60, 256, image);
Assert.That.ImageRegionIsColor(Rectangle.FromLTRB(0, 0, 60, 256), Color.Black, (Bitmap)image);
}
}
/// <summary>
/// ONLY Use this concatenation method when you want to concatenate the files for a fixed single day.
/// The files to be concatenated must be somewhere in the subdirectory structure of the passed list of data directories
/// Read them into a dictionary
/// MOST RECENT METHOD TO CONCATENATE Spectral INDEX.CSV FILES - Early September 2015.
/// It is designed to deal with Yvonne's case where want to concatenate files distributed over arbitrary directories.
/// It only merges files for the passed fixed date. i.e only 24 hours
/// </summary>
public static void DrawSpectralIndexFiles(
Dictionary <string, double[, ]> dictionary,
LdSpectrogramConfig sgConfig,
IndexGenerationData indexGenerationData,
FileInfo indexPropertiesConfigFileInfo,
DirectoryInfo opDir,
SiteDescription siteDescription,
FileInfo sunriseDataFile = null,
List <GapsAndJoins> segmentErrors = null)
{
// derive new indices such as sqrt(PMN), NCDI etc -- main reason for this is to view what their distributions look like.
dictionary = IndexMatrices.AddDerivedIndices(dictionary);
// Calculate the index distribution statistics and write to a json file. Also save as png image
if (indexGenerationData.RecordingStartDate != null)
{
DateTimeOffset dto = (DateTimeOffset)indexGenerationData.RecordingStartDate;
string dateString = $"{dto.Year}{dto.Month:D2}{dto.Day:D2}";
string opFileStem = $"{siteDescription.SiteName}_{dateString}";
var indexDistributions = IndexDistributions.WriteSpectralIndexDistributionStatistics(dictionary, opDir, opFileStem);
//SummaryIndexBase[] summaryIndices = null;
string analysisType = "Towsey.Acoustic";
Tuple <Image, string>[] tuple = LDSpectrogramRGB.DrawSpectrogramsFromSpectralIndices(
opDir, // topLevelDirectories[0], // this should not be required but it is - because things have gotten complicated !
opDir,
sgConfig,
indexPropertiesConfigFileInfo,
indexGenerationData,
opFileStem,
analysisType,
dictionary,
null, //summaryIndices,
indexDistributions,
siteDescription,
sunriseDataFile,
segmentErrors,
ImageChrome.With);
}
}
public void SummariseResults(AnalysisSettings settings, FileSegment inputFileSegment, EventBase[] events, SummaryIndexBase[] indices, SpectralIndexBase[] spectralIndices, AnalysisResult2[] results)
{
var acousticIndicesConfig = (AcousticIndicesConfig)settings.AnalysisAnalyzerSpecificConfiguration;
var sourceAudio = inputFileSegment.Source;
var resultsDirectory = AnalysisCoordinator.GetNamedDirectory(settings.AnalysisOutputDirectory, this);
bool tileOutput = acousticIndicesConfig.TileOutput;
var frameWidth = acousticIndicesConfig.FrameLength;
int sampleRate = AppConfigHelper.DefaultTargetSampleRate;
sampleRate = acousticIndicesConfig.ResampleRate ?? sampleRate;
// Gather settings for rendering false color spectrograms
var ldSpectrogramConfig = acousticIndicesConfig.LdSpectrogramConfig;
string basename = Path.GetFileNameWithoutExtension(sourceAudio.Name);
// output to disk (so other analyzers can use the data,
// only data - configuration settings that generated these indices
// this data can then be used by post-process analyses
/* NOTE: The value for FrameStep is used only when calculating a standard spectrogram
* FrameStep is NOT used when calculating Summary and Spectral indices.
*/
var indexConfigData = new IndexGenerationData()
{
RecordingExtension = inputFileSegment.Source.Extension,
RecordingBasename = basename,
RecordingStartDate = inputFileSegment.TargetFileStartDate,
RecordingDuration = inputFileSegment.TargetFileDuration.Value,
SampleRateOriginal = inputFileSegment.TargetFileSampleRate.Value,
SampleRateResampled = sampleRate,
FrameLength = frameWidth,
FrameStep = settings.Configuration.GetIntOrNull(AnalysisKeys.FrameStep) ?? frameWidth,
IndexCalculationDuration = acousticIndicesConfig.IndexCalculationDurationTimeSpan,
BgNoiseNeighbourhood = acousticIndicesConfig.BgNoiseBuffer,
AnalysisStartOffset = inputFileSegment.SegmentStartOffset ?? TimeSpan.Zero,
MaximumSegmentDuration = settings.AnalysisMaxSegmentDuration,
BackgroundFilterCoeff = SpectrogramConstants.BACKGROUND_FILTER_COEFF,
LongDurationSpectrogramConfig = ldSpectrogramConfig,
};
var icdPath = FilenameHelpers.AnalysisResultPath(
resultsDirectory,
basename,
IndexGenerationData.FileNameFragment,
"json");
Json.Serialise(icdPath.ToFileInfo(), indexConfigData);
// gather spectra to form spectrograms. Assume same spectra in all analyzer results
// this is the most efficient way to do this
// gather up numbers and strings store in memory, write to disk one time
// this method also AUTOMATICALLY SORTS because it uses array indexing
var dictionaryOfSpectra = spectralIndices.ToTwoDimensionalArray(SpectralIndexValues.CachedSelectors, TwoDimensionalArray.Rotate90ClockWise);
// Calculate the index distribution statistics and write to a json file. Also save as png image
var indexDistributions = IndexDistributions.WriteSpectralIndexDistributionStatistics(dictionaryOfSpectra, resultsDirectory, basename);
// HACK: do not render false color spectrograms unless IndexCalculationDuration = 60.0 (the normal resolution)
if (acousticIndicesConfig.IndexCalculationDurationTimeSpan != 60.0.Seconds())
{
Log.Warn("False color spectrograms were not rendered");
}
else
{
FileInfo indicesPropertiesConfig = acousticIndicesConfig.IndexPropertiesConfig.ToFileInfo();
// Actually draw false color / long duration spectrograms
Tuple <Image <Rgb24>, string>[] images =
LDSpectrogramRGB.DrawSpectrogramsFromSpectralIndices(
inputDirectory: resultsDirectory,
outputDirectory: resultsDirectory,
ldSpectrogramConfig: ldSpectrogramConfig,
indexPropertiesConfigPath: indicesPropertiesConfig,
indexGenerationData: indexConfigData,
basename: basename,
analysisType: this.Identifier,
indexSpectrograms: dictionaryOfSpectra,
indexStatistics: indexDistributions,
imageChrome: (!tileOutput).ToImageChrome());
if (tileOutput)
{
Debug.Assert(images.Length == 2);
Log.Info("Tiling output at scale: " + acousticIndicesConfig.IndexCalculationDuration);
foreach (var image in images)
{
TileOutput(resultsDirectory, Path.GetFileNameWithoutExtension(sourceAudio.Name), image.Item2 + ".Tile", inputFileSegment, image.Item1);
}
}
}
}
public static void Execute(Arguments arguments)
{
if (arguments == null)
{
throw new NoDeveloperMethodException();
}
string date = "# DATE AND TIME: " + DateTime.Now;
LoggedConsole.WriteLine("# DRAW LONG DURATION SPECTROGRAMS DERIVED FROM CSV FILES OF SPECTRAL INDICES OBTAINED FROM AN AUDIO RECORDING");
LoggedConsole.WriteLine(date);
LoggedConsole.WriteLine("# Spectrogram Config file: " + arguments.FalseColourSpectrogramConfig);
LoggedConsole.WriteLine("# Index Properties Config file: " + arguments.IndexPropertiesConfig);
LoggedConsole.WriteLine();
(FileInfo indexGenerationDataFile, FileInfo indexDistributionsFile) =
ZoomParameters.CheckNeededFilesExist(arguments.InputDataDirectory.ToDirectoryInfo());
var indexGenerationData = Json.Deserialize <IndexGenerationData>(indexGenerationDataFile);
// spectral distribution statistics is required only when calcualting difference spectrograms.
Dictionary <string, IndexDistributions.SpectralStats> indexDistributionsData = null;
if (indexDistributionsFile != null && indexDistributionsFile.Exists)
{
indexDistributionsData = IndexDistributions.Deserialize(indexDistributionsFile);
}
// this config can be found in IndexGenerationData. If config argument not specified, simply take it from icd file
LdSpectrogramConfig config;
if (arguments.FalseColourSpectrogramConfig == null)
{
config = indexGenerationData.LongDurationSpectrogramConfig;
}
else
{
config = LdSpectrogramConfig.ReadYamlToConfig(arguments.FalseColourSpectrogramConfig.ToFileInfo());
}
FilenameHelpers.ParseAnalysisFileName(indexGenerationDataFile, out var originalBaseName, out var _, out var _);
// CHECK FOR ERROR SEGMENTS - get zero signal array
var input = arguments.InputDataDirectory.ToDirectoryInfo();
var csvFile = new FileInfo(Path.Combine(input.FullName, originalBaseName + "__Towsey.Acoustic.Indices.csv"));
//Dictionary<string, double[]> summaryIndices = CsvTools.ReadCSVFile2Dictionary(csvFile.FullName);
//var summaryIndices = Csv.ReadFromCsv<Dictionary<string, double[]>>(csvFile);
var summaryIndices = Csv.ReadFromCsv <SummaryIndexValues>(csvFile);
var indexErrors = GapsAndJoins.DataIntegrityCheckForZeroSignal(summaryIndices);
//config.IndexCalculationDuration = TimeSpan.FromSeconds(1.0);
//config.XAxisTicInterval = TimeSpan.FromSeconds(60.0);
//config.IndexCalculationDuration = TimeSpan.FromSeconds(60.0);
//config.XAxisTicInterval = TimeSpan.FromSeconds(3600.0);
LDSpectrogramRGB.DrawSpectrogramsFromSpectralIndices(
inputDirectory: input,
outputDirectory: arguments.OutputDirectory.ToDirectoryInfo(),
ldSpectrogramConfig: config,
indexPropertiesConfigPath: arguments.IndexPropertiesConfig.ToFileInfo(),
indexGenerationData: indexGenerationData,
basename: originalBaseName,
analysisType: AcousticIndices.TowseyAcoustic,
indexSpectrograms: null,
indexStatistics: indexDistributionsData,
segmentErrors: indexErrors,
imageChrome: false.ToImageChrome());
Log.Success("Draw Long Duration Spectrograms complete!");
}
public void TestAnalyzeSr64000Recording()
{
int sampleRate = 64000;
double duration = 420; // signal duration in seconds = 7 minutes
int[] harmonics = { 500, 1000, 2000, 4000, 8000 };
var recording = DspFilters.GenerateTestRecording(sampleRate, duration, harmonics, WaveType.Cosine);
string recordingName = "TemporaryRecording2";
var recordingPath = this.outputDirectory.CombineFile(recordingName + ".wav");
WavWriter.WriteWavFileViaFfmpeg(recordingPath, recording.WavReader);
var fst = FreqScaleType.Linear125Octaves7Tones28Nyquist32000;
var freqScale = new FrequencyScale(fst);
/*
* // draw the signal as spectrogram just for debugging purposes
* // but can only draw a two minute spectrogram when sr=64000 - change duration above.
* duration = 120; // if drawing sonogram, then set signal duration = 2 minutes
* var sonogram = OctaveFreqScale.ConvertRecordingToOctaveScaleSonogram(recording, fst);
* var sonogramImage = sonogram.GetImageFullyAnnotated(sonogram.GetImage(), "SPECTROGRAM", freqScale.GridLineLocations);
* var outputImagePath = this.outputDirectory.CombineFile("SignalSpectrogram_OctaveFreqScale.png");
* sonogramImage.Save(outputImagePath.FullName);
*/
// Now need to rewrite the config file with new parameter settings
var configPath = PathHelper.ResolveConfigFile("Towsey.Acoustic.yml");
// Convert the Config config to IndexCalculateConfig class and merge in the unnecesary parameters.
//Config configuration = Yaml.Deserialise(configPath);
//IndexCalculateConfig config = IndexCalculateConfig.GetConfig(configuration, false);
// because of difficulties in dealing with Config config files, just edit the text file!!!!!
var configLines = File.ReadAllLines(configPath.FullName);
configLines[configLines.IndexOf(x => x.StartsWith("IndexCalculationDuration: "))] = "IndexCalculationDuration: 15.0";
//configLines[configLines.IndexOf(x => x.StartsWith("BgNoiseBuffer: "))] = "BgNoiseBuffer: 5.0";
configLines[configLines.IndexOf(x => x.StartsWith("FrequencyScale: Linear"))] = "FrequencyScale: " + fst;
// the is the only octave scale currently functioning for IndexCalculate class
configLines[configLines.IndexOf(x => x.StartsWith("FrameLength"))] = $"FrameLength: {freqScale.WindowSize}";
configLines[configLines.IndexOf(x => x.StartsWith("ResampleRate: "))] = "ResampleRate: 64000";
// write the edited Config file to temporary output directory
var newConfigPath = this.outputDirectory.CombineFile("Towsey.Acoustic.yml");
File.WriteAllLines(newConfigPath.FullName, configLines);
PathHelper.ResolveConfigFile("IndexPropertiesConfig.yml").CopyTo(this.outputDirectory.CombineFile("IndexPropertiesConfig.yml").FullName);
var arguments = new AnalyseLongRecording.Arguments
{
Source = recordingPath,
Config = newConfigPath.FullName,
Output = this.outputDirectory,
MixDownToMono = true,
Parallel = !Debugger.IsAttached,
};
AnalyseLongRecording.Execute(arguments);
var resultsDirectory = this.outputDirectory.Combine("Towsey.Acoustic");
var listOfFiles = resultsDirectory.EnumerateFiles().ToArray();
Assert.AreEqual(19, listOfFiles.Length);
var csvCount = listOfFiles.Count(f => f.Name.EndsWith(".csv"));
Assert.AreEqual(15, csvCount);
var jsonCount = listOfFiles.Count(f => f.Name.EndsWith(".json"));
Assert.AreEqual(2, jsonCount);
var pngCount = listOfFiles.Count(f => f.Name.EndsWith(".png"));
Assert.AreEqual(2, pngCount);
var bgnFile = resultsDirectory.CombineFile(recordingName + "__Towsey.Acoustic.BGN.csv");
double[,] actualBgn = Csv.ReadMatrixFromCsv <double>(bgnFile, TwoDimensionalArray.None);
var expectedSpectrumFile = PathHelper.ResolveAsset("LongDuration", "BgnMatrix.OctaveScale.csv");
// uncomment the following line when first produce the array
// bgnFile.CopyTo(expectedSpectrumFile.FullName);
// compare actual BGN file with expected file.
var expectedBgn = Csv.ReadMatrixFromCsv <double>(expectedSpectrumFile, TwoDimensionalArray.None);
CollectionAssert.That.AreEqual(expectedBgn, actualBgn, 0.000_000_001);
var array = MatrixTools.GetRow(actualBgn, 0);
Assert.AreEqual(28, actualBgn.RowLength());
Assert.AreEqual(256, array.Length);
// draw array just to check peaks are in correct places - just for debugging purposes
var ldsBgnSpectrumFile = this.outputDirectory.CombineFile("Spectrum2.png");
GraphsAndCharts.DrawGraph(array, "LD BGN SPECTRUM Octave", ldsBgnSpectrumFile);
// ##########################################
// SECOND part of test is to create the LD spectrograms because they are not created when IndexCalcDuration < 60 seconds
// first read in the index generation data
var icdPath = resultsDirectory.CombineFile(recordingName + "__IndexGenerationData.json");
var indexConfigData = Json.Deserialize <IndexGenerationData>(icdPath);
var indexPropertiesConfig = PathHelper.ResolveConfigFile("IndexPropertiesConfig.yml");
var ldSpectrogramConfigFile = PathHelper.ResolveConfigFile("SpectrogramFalseColourConfig.yml");
var ldSpectrogramConfig = LdSpectrogramConfig.ReadYamlToConfig(ldSpectrogramConfigFile);
ldSpectrogramConfig.FreqScale = fst.ToString();
// finally read in the dictionary of spectra
string analysisType = "Towsey.Acoustic";
var keys = LDSpectrogramRGB.GetArrayOfAvailableKeys();
var dictionaryOfSpectra = IndexMatrices.ReadSpectralIndices(resultsDirectory, recordingName, analysisType, keys);
LDSpectrogramRGB.DrawSpectrogramsFromSpectralIndices(
inputDirectory: resultsDirectory,
outputDirectory: resultsDirectory,
ldSpectrogramConfig: ldSpectrogramConfig,
indexPropertiesConfigPath: indexPropertiesConfig,
indexGenerationData: indexConfigData,
basename: recordingName,
analysisType: analysisType,
indexSpectrograms: dictionaryOfSpectra);
// test number of images - should now be 23
listOfFiles = resultsDirectory.EnumerateFiles().ToArray();
pngCount = listOfFiles.Count(f => f.Name.EndsWith(".png"));
Assert.AreEqual(22, pngCount);
var twoMapsImagePath = resultsDirectory.CombineFile(recordingName + "__2Maps.png");
var twoMapsImage = Image.Load <Rgb24>(twoMapsImagePath.FullName);
// image is (7*4) * 652
Assert.AreEqual(28, twoMapsImage.Width);
Assert.AreEqual(652, twoMapsImage.Height);
}
