public static void WriteTable2Console(DataTable dt)
{
if (dt == null)
{
return;
}
string[] headers = GetColumnNames(dt);
foreach (string name in headers)
{
LoggedConsole.Write(" {0,-10}", name);
}
LoggedConsole.WriteLine();
var rows = dt.Rows;
foreach (DataRow row in rows)
{
for (int i = 0; i < headers.Length; i++)
{
LoggedConsole.Write(" {0:f2}{1,-7}", row[headers[i]], " ");
}
LoggedConsole.WriteLine();
}
}
/// <summary>
/// This method was written 9th March 2015 to process a dataset of some 1000 x 5 second recordings.
/// The dataset was originally prepared by Meriem for use in her Master's thesis.
/// The data is being processed to produce grayscale spectrogram images for use by Mangalam.
/// She will classify them using a CNN.
/// Note: NO SNR statistics are calculated. All reocrdings must be in single level directory.
/// </summary>
public static void ProcessMeriemsDataset()
{
// 1. set up the necessary files
//FileInfo csvFileInfo = arguments.Source;
//FileInfo configFile = arguments.Config;
//DirectoryInfo output = arguments.Output;
DirectoryInfo inputDirInfo = new DirectoryInfo(@"C:\SensorNetworks\WavFiles\MeriemDataSet\1016 Distinct files");
FileInfo configFile = new FileInfo(@"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\Mangalam.Sonogram.yml");
DirectoryInfo outputDirInfo = new DirectoryInfo(@"C:\SensorNetworks\Output\ConvDNN\MeriemDataset\");
// 2. get the config dictionary
var configDict = GetConfigurationForConvCnn(configFile);
if (true)
{
// if VERBOSE
const string title = "# PRE-PROCESS SHORT AUDIO RECORDINGS FOR Convolutional DNN";
string date = "# DATE AND TIME: " + DateTime.Now;
LoggedConsole.WriteLine(title);
LoggedConsole.WriteLine(date);
// print out the parameters
LoggedConsole.WriteLine("\nPARAMETERS");
foreach (var kvp in configDict)
{
LoggedConsole.WriteLine("{0} = {1}", kvp.Key, kvp.Value);
}
}
int fileCount = 0;
FileInfo[] inputFiles = inputDirInfo.GetFiles();
foreach (var file in inputFiles)
{
LoggedConsole.Write(".");
if (!file.Exists)
{
continue;
}
fileCount++;
// need to set up the config with file names
configDict[ConfigKeys.Recording.Key_RecordingFileName] = file.FullName;
configDict[ConfigKeys.Recording.Key_RecordingCallName] = Path.GetFileNameWithoutExtension(file.FullName);
// reset the frame size
configDict["FrameLength"] = "512";
//AudioToSonogramResult result = GenerateFourSpectrogramImages(file, configDict, outputDirInfo);
GenerateSpectrogramImages(file, configDict, outputDirInfo);
fileCount++;
} // end foreach()
LoggedConsole.WriteLine("\nFile Count =" + fileCount);
LoggedConsole.WriteLine("\n##### FINISHED ############################\n");
}
public static void DisplaySignal(double[] sig)
{
double[] newSig = DataTools.normalise(sig);
foreach (double value in newSig)
{
int count = (int)(value * 50);
for (int i = 0; i < count; i++)
{
LoggedConsole.Write("=");
}
LoggedConsole.WriteLine("=");
}
}
//***************************************************************************************************************************************
//***************************************************************************************************************************************
//***************************************** STATIC METHODS ************************************************************************
//***************************************************************************************************************************************
//***************************************************************************************************************************************
/// <summary>
/// Need to allow for possibility that a wt vector = null.
/// </summary>
public static void DisplayClusterWeights(List <double[]> clusterWts, int[] clusterHits)
{
int clusterCount = 0;
LoggedConsole.WriteLine(" wts wtSum\t wins");
for (int i = 0; i < clusterWts.Count; i++)
{
int wins = 0;
LoggedConsole.Write("wts{0:D3} ", i + 1); //write the cluster number
if (clusterWts[i] == null)
{
for (int j = 0; j < 32; j++)
{
LoggedConsole.Write(" ");
}
LoggedConsole.WriteLine(" null");
}
else
{
for (int j = 0; j < clusterWts[i].Length; j++)
{
if (clusterWts[i][j] > 0.0)
{
LoggedConsole.Write("1");
}
else
{
LoggedConsole.Write("0");
}
}
for (int j = 0; j < clusterHits.Length; j++)
{
if (clusterHits[j] == i)
{
wins++;
}
}
LoggedConsole.WriteLine(" {0}\t\t{1}", clusterWts[i].Sum(), wins);
clusterCount++;
}
}
LoggedConsole.WriteLine("Cluster Count = {0}", clusterCount);
} // end DisplayClusterWeights()
} // GetInstanceRepresentations()
public static void GetSpeciesRepresentations(Arguments arguments, Output output)
{
LoggedConsole.WriteLine("\n\n2a. Obtain feature representation of every species.");
int instanceCount = arguments.InstanceCount;
int speciesCount = arguments.SpeciesCount;
var keyArray = FEATURE_KEYS.Split(',');
int featureCount = output.InstanceFeatureMatrix.GetLength(1);
// initialise species description matrix
double[,] speciesFeatureMatrix = new double[speciesCount, featureCount];
int[] frameNumbersPerSpecies = new int[speciesCount];
// loop through all 50 species
for (int i = 0; i < speciesCount; i++)
{
int speciesLabel = i + 1;
LoggedConsole.Write(" " + speciesLabel);
// loop through all instances multiple times - once for each species
for (int j = 0; j < instanceCount; j++)
{
if (output.SpeciesID[j] != speciesLabel)
{
continue;
}
//aggregate the instance feature values
double[] ipVector = MatrixTools.GetRow(output.InstanceFeatureMatrix, j);
for (int c = 0; c < featureCount; c++)
{
speciesFeatureMatrix[i, c] += ipVector[c];
}
//output.InstanceNumbersPerSpecies[i]++;
frameNumbersPerSpecies[i] += output.FrameNumbersPerInstance[j];
} // end for loop j over all instances
} // loop through all 50 species
LoggedConsole.WriteLine(" Done");
output.SpeciesFeatureMatrix = speciesFeatureMatrix;
output.FrameNumbersPerSpecies = frameNumbersPerSpecies;
} // GetSpeciesRepresentations()
public static void DisplaySignal(double[] sig, bool showIndex)
{
double[] newSig = DataTools.normalise(sig);
for (int n = 0; n < sig.Length; n++)
{
if (showIndex)
{
LoggedConsole.Write(n.ToString("D3") + "|");
}
int count = (int)(newSig[n] * 50);
for (int i = 0; i < count; i++)
{
LoggedConsole.Write("=");
}
LoggedConsole.WriteLine("=");
}
}
public static List <string> GetDataSlice(DirectoryInfo dataTableDir, string key, int value)
{
LoggedConsole.Write("GetDataSlice() " + key + "=" + value + ": ");
List <string> outputList = new List <string>();
FileInfo[] fileList = dataTableDir.GetFiles();
foreach (FileInfo file in fileList)
{
Console.Write("."); // so impatient user knows something is happening!
List <string> list = FileTools.ReadSelectedLinesOfCsvFile(file.FullName, key, value);
if (list == null)
{
continue;
}
outputList.AddRange(list);
}
LoggedConsole.WriteLine();
return(outputList);
}
public static void WriteDistribution(int[] distribution, double binWidth)
{
int binCount = distribution.Length; // number of bins
LoggedConsole.WriteLine("\nDistribution over " + binCount + " bins");
for (int i = 0; i < binCount; i++)
{
LoggedConsole.Write(i + "\t");
}
LoggedConsole.WriteLine();
for (int i = 0; i < binCount; i++)
{
LoggedConsole.Write((int)(i * binWidth) + "\t");
}
LoggedConsole.WriteLine("(Total " + (int)(binWidth * binCount) + " bins)");
for (int i = 0; i < binCount; i++)
{
LoggedConsole.Write(distribution[i] + "\t");
}
LoggedConsole.WriteLine("Total instances=" + DataTools.Sum(distribution));
}
/// <summary>
/// Use this method to find match in sonogram to a symbolic definition of a bird call.
/// That is, the template should be matrix of binary or trinary values.
/// </summary>
/// <param name="dBThreshold">Not used in calculation. Only used to speed up loop over the spectrogram.</param>
public static Tuple <double[]> Execute_Bi_or_TrinaryMatch(double[,] template, SpectrogramStandard sonogram,
List <AcousticEvent> segments, int minHz, int maxHz, double dBThreshold)
{
Log.WriteLine("SEARCHING FOR EVENTS LIKE TARGET.");
if (segments == null)
{
return(null);
}
int minBin = (int)(minHz / sonogram.FBinWidth);
int maxBin = (int)(maxHz / sonogram.FBinWidth);
int templateHeight = template.GetLength(0);
int templateWidth = template.GetLength(1);
int cellCount = templateHeight * templateWidth;
//var image = BaseSonogram.Data2ImageData(target);
//ImageTools.DrawMatrix(image, 1, 1, @"C:\SensorNetworks\Output\FELT_Currawong\target.png");
// ######### Following line normalises template scores for comparison between templates.
// ######### Ensures OP=0 for featureless sonogram #########
// ######### template score = (average dB of on-template-cells - average dB of off-template-cells).
WriteTemplate2Console(template);
var tuple1 = NormaliseBiTrinaryMatrix(template);
template = tuple1.Item1;
int positiveCount = tuple1.Item2;
int negativeCount = tuple1.Item3;
Log.WriteLine("TEMPLATE: Number of POS cells/total cells = {0}/{1}", positiveCount, cellCount);
Log.WriteLine("TEMPLATE: Number of NEG cells/total cells = {0}/{1}", negativeCount, cellCount);
double[] scores = new double[sonogram.FrameCount];
foreach (AcousticEvent av in segments)
{
Log.WriteLine("SEARCHING SEGMENT.");
int startRow = (int)Math.Floor(av.TimeStart * sonogram.FramesPerSecond);
int endRow = (int)Math.Floor(av.TimeEnd * sonogram.FramesPerSecond);
if (endRow >= sonogram.FrameCount)
{
endRow = sonogram.FrameCount;
}
int stopRow = endRow - templateWidth - 1;
if (stopRow <= startRow)
{
stopRow = startRow + 1; //want minimum of one row
}
for (int r = startRow; r < stopRow; r++)
{
double max = -double.MaxValue;
//int maxOnCount= 0; //used to display % ON-count and maybe to modify the score.
int binBuffer = 10;
for (int bin = -binBuffer; bin < +binBuffer; bin++)
{
int c = minBin + bin;
if (c < 0)
{
c = 0;
}
double crossCor = 0.0;
//int onCount = 0;
for (int j = 0; j < templateWidth; j++)
{
int c0 = c + templateHeight - 1;
for (int i = 0; i < templateHeight; i++)
{
crossCor += sonogram.Data[r + j, c0 - i] * template[i, j];
//if ((sonogram.Data[r + j, c0 - i] > 0.0) && (template[i, j] > 0)) onCount++;
}
}
//var image = BaseSonogram.Data2ImageData(matrix);
//ImageTools.DrawMatrix(image, 1, 1, @"C:\SensorNetworks\Output\FELT_CURLEW\compare.png");
if (crossCor > max)
{
max = crossCor;
//maxOnCount = onCount;
}
} // end freq bins
//following line yields score = av of PosCells - av of NegCells.
scores[r] = max / positiveCount;
// display percent onCount
//int pcOnCount = maxOnCount * 100 / positiveCount;
//if (r % 100 == 0) { LoggedConsole.WriteLine("{0} - {1:f3}", r, scores[r]); }
//if (scores[r] >= dBThreshold) { LoggedConsole.WriteLine("r={0} score={1} %on={2}.", r, scores[r], pcOnCount); }
if (r % 100 == 0)
{
LoggedConsole.Write(".");
}
if (scores[r] < dBThreshold)
{
r += 3; //skip where score is low
}
} // end of rows in segment
LoggedConsole.WriteLine("\nFINISHED SEARCHING SEGMENT FOR ACOUSTIC EVENT.");
} // foreach (AcousticEvent av in segments)
var tuple = Tuple.Create(scores);
return(tuple);
}
/// <summary>
/// Use this method to find match in sonogram to a symbolic definition of a bird call.
/// That is, the template should be matrix of binary or trinary values.
/// </summary>
public static Tuple <double[]> Execute_Spr_Match(char[,] template, SpectrogramStandard sonogram,
List <AcousticEvent> segments, int minHz, int maxHz, double dBThreshold)
{
int lineLength = 10;
//dBThreshold = 9;
Log.WriteLine("SEARCHING FOR EVENTS LIKE TARGET (SPR).");
if (segments == null)
{
return(null);
}
int minBin = (int)(minHz / sonogram.FBinWidth);
int maxBin = (int)(maxHz / sonogram.FBinWidth);
int templateFrames = template.GetLength(0); // time axis - SPR template is same orientation as the sonogram data matrix
int templateFreqBins = template.GetLength(1); // freq axis
int cellCount = templateFrames * templateFreqBins;
int positiveCount = SprTools.CountTemplateChars(template);
int negativeCount = cellCount - positiveCount;
Log.WriteLine("TEMPLATE: Number of POS cells/total cells = {0}/{1}", positiveCount, cellCount);
// Log.WriteLine("TEMPLATE: Number of NEG cells/total cells = {0}/{1}", negativeCount, cellCount);
char[,] charogram = SprTools.Target2SymbolicTracks(sonogram.Data, dBThreshold, lineLength);
//var m = DataTools.MatrixTranspose(charogram);
FileTools.WriteMatrix2File(charogram, "C:\\SensorNetworks\\Output\\FELT_MultiOutput_5templates\\char_ogram.txt"); //view the char-ogram
double[] scores = new double[sonogram.FrameCount];
foreach (AcousticEvent av in segments)
{
Log.WriteLine("SEARCHING SEGMENT.");
int startRow = (int)Math.Floor(av.TimeStart * sonogram.FramesPerSecond);
int endRow = (int)Math.Floor(av.TimeEnd * sonogram.FramesPerSecond);
if (endRow >= sonogram.FrameCount)
{
endRow = sonogram.FrameCount;
}
int stopRow = endRow - templateFrames - 1;
if (stopRow <= startRow)
{
stopRow = startRow + 1; //want minimum of one row
}
for (int r = startRow; r < stopRow; r++)
{
double maxSimilarity = -double.MaxValue;
int binBuffer = 10;
// calculate similarity at one frame position
for (int bin = -binBuffer; bin < +binBuffer; bin++)
{
int c = minBin + bin;
if (c < 0)
{
c = 0;
}
double onSum = 0.0;
double offSum = 0.0;
// calculate onSum and offSum
// freq axis
for (int j = 0; j < templateFreqBins; j++)
{
for (int i = 0; i < templateFrames; i++)
{
if (charogram[r + i, c + j] == '-')
{
continue;
}
else
if (template[i, j] == '-')
{
offSum += sonogram.Data[r + i, c + j];
}
else
{
//char ColumnLeft = charogram[r + i, c + j];
//char ColumnRight = template[i, j];
//int difference = (int)ColumnLeft - (int)ColumnRight;
int diff = SprTools.SymbolDifference(charogram[r + i, c + j], template[i, j]);
onSum += (90 - diff) / (double)90 * sonogram.Data[r + i, c + j];
}
}
} // calculate similarity
double similarity = (onSum / positiveCount) - (offSum / negativeCount);
if (similarity > maxSimilarity)
{
maxSimilarity = similarity;
}
} // end freq bins
//following line yields score = av of PosCells - av of NegCells.
scores[r] = maxSimilarity;
//if (r % 100 == 0) { LoggedConsole.WriteLine("{0} - {1:f3}", r, scores[r]); }
//if (scores[r] >= dBThreshold) { LoggedConsole.WriteLine("r={0} score={1}.", r, scores[r]); }
if (r % 100 == 0)
{
LoggedConsole.Write(".");
}
if (scores[r] < dBThreshold)
{
r += 3; //skip where score is low
}
} // end of rows in segment
LoggedConsole.WriteLine("\nFINISHED SEARCHING SEGMENT FOR ACOUSTIC EVENT.");
} // foreach (AcousticEvent av in segments)
var tuple = Tuple.Create(scores);
return(tuple);
} //Execute_Spr_Match()
} //Execute()
public static Output GetInstanceRepresentations(Arguments arguments)
{
LoggedConsole.WriteLine("1. Read in all Instances and do feature extraction");
//################################### FEATURE WEIGHTS
//TRY DIFFERENT WEIGHTINGS assuming following "SPT,RHZ,RVT,RPS,RNG";
bool doDeltaFeatures = false;
double[] weights = { 1.0, 1.0, 0.8, 0.7, 0.7 };
double[] deltaWeights = { 1.0, 1.0, 0.8, 0.7, 0.7, 0.5, 0.4, 0.4, 0.2, 0.2 };
if (doDeltaFeatures)
{
weights = deltaWeights;
}
//MAX-POOLING for SPECTRAL REDUCTION
// frequency bins used to reduce dimensionality of the 256 spectral values.
int startBin = 8;
int maxOf2Bin = 117;
int maxOf3Bin = 160;
int endBin = 200;
double[] testArray = new double[256];
for (int i = 0; i < testArray.Length; i++)
{
testArray[i] = i;
}
double[] reducedArray = MaxPoolingLimited(testArray, startBin, maxOf2Bin, maxOf3Bin, endBin);
int reducedSpectralLength = reducedArray.Length;
LoggedConsole.WriteLine(" Reduced spectral length = " + reducedSpectralLength);
int instanceCount = arguments.InstanceCount;
int speciesCount = arguments.SpeciesCount;
// READ IN THE SPECIES LABELS FILE AND SET UP THE DATA
string[] fileID = new string[instanceCount];
int[] speciesID = new int[speciesCount];
ReadGlotinsSpeciesLabelFile(arguments.SpeciesLabelsFile, instanceCount, out fileID, out speciesID);
// INIT array of species counts
int[] instanceNumbersPerSpecies = new int[speciesCount];
// INIT array of frame counts
int[] frameNumbersPerInstance = new int[instanceCount];
// initialise species description matrix
var keyArray = FEATURE_KEYS.Split(',');
int totalFeatureCount = keyArray.Length * reducedArray.Length;
Console.WriteLine(" Total Feature Count = " + totalFeatureCount);
if (doDeltaFeatures)
{
totalFeatureCount *= 2;
LoggedConsole.WriteLine(" Total Delta Feature Count = " + totalFeatureCount);
}
// one matrix row per species
double[,] instanceFeatureMatrix = new double[instanceCount, totalFeatureCount];
// loop through all all instances
for (int j = 0; j < instanceCount; j++)
{
LoggedConsole.Write(".");
int frameCount = 0;
// get the spectral index files
int speciesLabel = speciesID[j];
// dictionary to store feature spectra for instance.
var aggreDictionary = new Dictionary <string, double[]>();
// dictionary to store delta spectra for instance.
var deltaDictionary = new Dictionary <string, double[]>();
foreach (string key in keyArray)
{
string name = string.Format("{0}_Species{1:d2}.{2}.csv", fileID[j], speciesLabel, key);
FileInfo file = new FileInfo(Path.Combine(arguments.InputDataDirectory.FullName, name));
if (file.Exists)
{
int binCount;
double[,] matrix = IndexMatrices.ReadSpectrogram(file, out binCount);
// create or get the array of spectral values.
double[] aggregateArray = new double[reducedSpectralLength];
double[] deltaArray = new double[reducedSpectralLength];
double[] ipVector = MatrixTools.GetRow(matrix, 0);
ipVector = DataTools.SubtractValueAndTruncateToZero(ipVector, arguments.BgnThreshold);
reducedArray = MaxPoolingLimited(ipVector, startBin, maxOf2Bin, maxOf3Bin, endBin);
double[] previousArray = reducedArray;
// transfer spectral values to array.
int rowCount = matrix.GetLength(0);
//rowCount = (int)Math.Round(rowCount * 0.99); // ###################### USE ONLY 99% of instance
//if (rowCount > 1200) rowCount = 1200;
for (int r = 1; r < rowCount; r++)
{
ipVector = MatrixTools.GetRow(matrix, r);
ipVector = DataTools.SubtractValueAndTruncateToZero(ipVector, arguments.BgnThreshold);
reducedArray = MaxPoolingLimited(ipVector, startBin, maxOf2Bin, maxOf3Bin, endBin);
for (int c = 0; c < reducedSpectralLength; c++)
{
aggregateArray[c] += reducedArray[c];
// Calculate the DELTA values TWO OPTIONS ##################################################
double delta = Math.Abs(reducedArray[c] - previousArray[c]);
//double delta = reducedArray[c] - previousArray[c];
//if (delta < 0.0) delta = 0.0;
//double delta = previousArray[c]; //previous array - i.e. do not calculate delta
deltaArray[c] += delta;
}
previousArray = reducedArray;
}
aggreDictionary[key] = aggregateArray;
deltaDictionary[key] = deltaArray;
frameCount = rowCount;
} //if (file.Exists)
} //foreach (string key in keyArray)
instanceNumbersPerSpecies[speciesLabel - 1]++;
frameNumbersPerInstance[j] += frameCount;
// create the matrix of instance descriptions which consists of concatenated vectors
// j = index of instance ID = row number
int featureID = 0;
foreach (string key in keyArray)
{
int featureOffset = featureID * reducedSpectralLength;
for (int c = 0; c < reducedSpectralLength; c++)
{
// TWO OPTIONS: SUM OR AVERAGE ######################################
//instanceFeatureMatrix[j, featureOffset + c] = dictionary[key][c];
instanceFeatureMatrix[j, featureOffset + c] = aggreDictionary[key][c] / frameCount;
}
featureID++;
}
if (doDeltaFeatures)
{
foreach (string key in keyArray)
{
int featureOffset = featureID * reducedSpectralLength;
for (int c = 0; c < reducedSpectralLength; c++)
{
// TWO OPTIONS: SUM OR AVERAGE ######################################
//instanceFeatureMatrix[j, featureOffset + c] = dictionary[key][c];
instanceFeatureMatrix[j, featureOffset + c] = deltaDictionary[key][c] / frameCount;
}
featureID++;
}
} // if doDeltaFeatures
} // end for loop j over all instances
LoggedConsole.WriteLine("Done!");
LoggedConsole.WriteLine("\nSum of species number array = " + instanceNumbersPerSpecies.Sum());
LoggedConsole.WriteLine("Sum of frame number array = " + frameNumbersPerInstance.Sum());
bool addLineNumbers = true;
string countsArrayOutputFilePath = Path.Combine(arguments.OutputDirectory.FullName, "BirdClef50_training_Counts.txt");
FileTools.WriteArray2File(instanceNumbersPerSpecies, addLineNumbers, countsArrayOutputFilePath);
// Initialise output data arrays
Output output = new Output();
output.FileID = fileID;
output.SpeciesID = speciesID;
output.InstanceNumbersPerSpecies = instanceNumbersPerSpecies;
output.ReducedSpectralLength = reducedSpectralLength;
// INIT array of frame counts
output.FrameNumbersPerInstance = frameNumbersPerInstance;
// matrix: each row= one instance; each column = one feature
output.InstanceFeatureMatrix = instanceFeatureMatrix;
output.Weights = weights;
return(output);
} // GetInstanceRepresentations()
public static async Task <int> Execute(RibbonPlot.Arguments arguments)
{
if (arguments.InputDirectories.IsNullOrEmpty())
{
throw new CommandLineArgumentException(
$"{nameof(arguments.InputDirectories)} is null or empty - please provide at least one source directory");
}
var doNotExist = arguments.InputDirectories.Where(x => !x.Exists);
if (doNotExist.Any())
{
throw new CommandLineArgumentException(
$"The following directories given to {nameof(arguments.InputDirectories)} do not exist: " + doNotExist.FormatList());
}
if (arguments.OutputDirectory == null)
{
arguments.OutputDirectory = arguments.InputDirectories.First();
Log.Warn(
$"{nameof(arguments.OutputDirectory)} was not provided and was automatically set to source directory {arguments.OutputDirectory}");
}
if (arguments.Midnight == null || arguments.Midnight == TimeSpan.Zero)
{
// we need this to be width of day and not zero for rounding functions later on
arguments.Midnight = RibbonPlotDomain;
Log.Debug($"{nameof(arguments.Midnight)} was reset to {arguments.Midnight}");
}
if (arguments.Midnight < TimeSpan.Zero || arguments.Midnight > RibbonPlotDomain)
{
throw new InvalidStartOrEndException($"{nameof(arguments.Midnight)} cannot be less than `00:00` or greater than `{RibbonPlotDomain}`");
}
LoggedConsole.Write("Begin scanning directories");
var allIndexFiles = arguments.InputDirectories.SelectMany(IndexGenerationData.FindAll);
if (allIndexFiles.IsNullOrEmpty())
{
throw new MissingDataException($"Could not find `{IndexGenerationData.FileNameFragment}` files in:" + arguments.InputDirectories.FormatList());
}
Log.Debug("Checking files have dates");
var indexGenerationDatas = allIndexFiles.Select(IndexGenerationData.Load);
var datedIndices = FileDateHelpers.FilterObjectsForDates(
indexGenerationDatas,
x => x.Source,
y => y.RecordingStartDate,
arguments.TimeSpanOffsetHint);
LoggedConsole.WriteLine($"{datedIndices.Count} index generation data files were loaded");
if (datedIndices.Count == 0)
{
throw new MissingDataException("No index generation files had dates, cannot proceed");
}
// now find the ribbon plots for these images - there are typically two color maps per index generation
var datesMappedToColorMaps = new Dictionary <string, Dictionary <DateTimeOffset, FileInfo> >(2);
foreach (var(date, indexData) in datedIndices)
{
Add(indexData.LongDurationSpectrogramConfig.ColorMap1);
Add(indexData.LongDurationSpectrogramConfig.ColorMap2);
void Add(string colorMap)
{
if (!datesMappedToColorMaps.ContainsKey(colorMap))
{
datesMappedToColorMaps.Add(colorMap, new Dictionary <DateTimeOffset, FileInfo>(datedIndices.Count));
}
// try to find the associated ribbon
var searchPattern = "*" + colorMap + LdSpectrogramRibbons.SpectralRibbonTag + "*";
if (Log.IsVerboseEnabled())
{
Log.Verbose($"Searching `{indexData.Source?.Directory}` with pattern `{searchPattern}`.");
}
var ribbonFile = indexData.Source?.Directory?.EnumerateFiles(searchPattern).FirstOrDefault();
if (ribbonFile == null)
{
Log.Warn($"Did not find expected ribbon file for color map {colorMap} in directory `{indexData.Source?.Directory}`."
+ "This can happen if the ribbon is missing or if more than one file matches the color map.");
}
datesMappedToColorMaps[colorMap].Add(date, ribbonFile);
}
}
// get the min and max dates and other things
var stats = new RibbonPlotStats(datedIndices, arguments.Midnight.Value);
Log.Debug($"Files found between {stats.Min:O} and {stats.Max:O}, rendering between {stats.Start:O} and {stats.End:O}, in {stats.Buckets} buckets");
bool success = false;
foreach (var(colorMap, ribbons) in datesMappedToColorMaps)
{
Log.Info($"Rendering ribbon plot for color map {colorMap}");
if (ribbons.Count(x => x.Value.NotNull()) == 0)
{
Log.Error($"There are no ribbon files found for color map {colorMap} - skipping this color map");
continue;
}
var image = CreateRibbonPlot(datedIndices, ribbons, stats);
var midnight = arguments.Midnight == RibbonPlotDomain
? string.Empty
: "Midnight=" + arguments.Midnight.Value.ToString("hhmm");
var path = FilenameHelpers.AnalysisResultPath(
arguments.OutputDirectory,
arguments.OutputDirectory.Name,
"RibbonPlot",
"png",
colorMap,
midnight);
using (var file = File.Create(path))
{
image.SaveAsPng(file);
}
image.Dispose();
success = true;
}
if (success == false)
{
throw new MissingDataException("Could not find any ribbon files for any of the color maps. No ribbon plots were produced.");
}
LoggedConsole.WriteSuccessLine("Completed");
return(ExceptionLookup.Ok);
}
public void Dispose()
{
Console.SetOut(this.originalConsoleOutput);
LoggedConsole.Write(this.ToString());
this.consoleOutput.Dispose();
}