public static void RecognizerScoresTest(string fileName, DirectoryInfo opDir, string testName, double[] scoreArray)
{
var testDir = new DirectoryInfo(opDir + $"\\UnitTest_{testName}");
var benchmarkDir = new DirectoryInfo(Path.Combine(testDir.FullName, ExpectedResultsDir));
if (!benchmarkDir.Exists)
{
benchmarkDir.Create();
}
var benchmarkFilePath = Path.Combine(benchmarkDir.FullName, fileName + ".TestScores.csv");
var testFilePath = Path.Combine(testDir.FullName, fileName + ".Scores.csv");
FileTools.WriteArray2File(scoreArray, testFilePath);
LoggedConsole.WriteLine($"# ARRAY TEST: Starting benchmark score array test for the {testName} recognizer:");
var benchmarkFile = new FileInfo(benchmarkFilePath);
if (!benchmarkFile.Exists)
{
LoggedConsole.WriteWarnLine(" A file of test scores does not exist. Writing output as future scores-test file");
FileTools.WriteArray2File(scoreArray, benchmarkFilePath);
}
else
{
CompareArrayWithBenchmark(testName, scoreArray, new FileInfo(benchmarkFilePath));
}
}
public static void TestLogging()
{
Contract.Requires <InvalidOperationException>(
configured,
"The logger system must be initialised before the logging can be tested");
var log = LogManager.GetLogger(nameof(Logging));
log.Prompt("Log test PROMPT");
log.Fatal("Log test FATAL");
log.Error("Log test ERROR");
log.Warn("Log test WARN");
log.Success("Log test SUCCESS");
log.Info("Log test INFO");
log.Debug("Log test DEBUG");
log.Trace("Log test TRACE");
log.Verbose("Log test VERBOSE");
LoggedConsole.WriteFatalLine("Clean wrapper FATAL", new Exception("I'm a fake"));
LoggedConsole.Log.Fatal("Clean log FATAL", new Exception("I'm a fake"));
LoggedConsole.WriteErrorLine("Clean wrapper ERROR");
LoggedConsole.Log.Error("Clean log ERROR");
LoggedConsole.WriteWarnLine("Clean wrapper WARN");
LoggedConsole.Log.Warn("Clean log WARN");
LoggedConsole.WriteSuccessLine("Clean wrapper SUCCESS");
LoggedConsole.Log.Success("Clean log SUCCESS");
LoggedConsole.WriteLine("Clean wrapper INFO");
LoggedConsole.Log.Info("Clean log INFO");
}
/// <summary>
/// This test checks a score array (array of doubles) against a standard or benchmark previously stored.
/// </summary>
public static void CompareArrayWithBenchmark(string testName, double[] scoreArray, FileInfo scoreFile)
{
LoggedConsole.WriteLine("# TESTING: Starting benchmark test for " + testName + ":");
LoggedConsole.WriteLine("# Comparing passed array of double with content of file <" + scoreFile.Name + ">");
bool allOk = true;
var scoreLines = FileTools.ReadTextFile(scoreFile.FullName);
if (scoreArray.Length != scoreLines.Count)
{
LoggedConsole.WriteWarnLine(" FAIL! SCORE ARRAY not same length as Benchmark.");
return;
}
for (int i = 0; i < scoreLines.Count; i++)
{
var str = scoreArray[i].ToString(CultureInfo.InvariantCulture);
if (!scoreLines[i].Equals(str))
{
LoggedConsole.WriteWarnLine($"Line {i}: {str} NOT= benchmark <{scoreLines[i]}>");
allOk = false;
}
}
if (allOk)
{
LoggedConsole.WriteSuccessLine(" SUCCESS! Passed the SCORE ARRAY TEST.");
}
else
{
LoggedConsole.WriteWarnLine(" FAILED THE SCORE ARRAY TEST");
}
LoggedConsole.WriteLine("Completed benchmark test.");
}
public override object ReadJson(
JsonReader reader,
Type objectType,
object existingValue,
JsonSerializer serializer)
{
JToken token = JToken.Load(reader);
if (token.Type == JTokenType.String)
{
var seconds = ((TimeSpan)token).TotalSeconds;
if (objectType.IsAssignableFrom(typeof(double)))
{
return(seconds);
}
else
{
LoggedConsole.WriteWarnLine("LegacyTimeSpanDataConverter is truncating values.");
return((int)seconds);
}
}
// otherwise fallback to default parsing semantics
return(token.ToObject(objectType));
}
public static void CompareTwoArrays(double[] array1, double[] array2)
{
LoggedConsole.WriteLine("# TESTING: Compare two arrays of double");
bool allOk = true;
if (array1.Length != array2.Length)
{
LoggedConsole.WriteWarnLine(" FAIL! ARRAYS are not of same length.");
return;
}
for (int i = 0; i < array1.Length; i++)
{
if (array1[i] != array2[i])
{
LoggedConsole.WriteWarnLine($"Line {i}: {array1[i]} != {array1[i]}");
allOk = false;
}
}
if (allOk)
{
LoggedConsole.WriteSuccessLine(" SUCCESS! Two arrays are equal.");
}
else
{
LoggedConsole.WriteWarnLine(" FAILED THE ARRAY TEST");
}
LoggedConsole.WriteLine("Completed test.");
}
public static System.Random GetRandom(int?seed = null)
{
seed = seed ?? Environment.TickCount;
Trace.WriteLine("\n\nRandom seed used: " + seed.Value);
LoggedConsole.WriteWarnLine($"Random seed: {seed}");
return(new System.Random(seed.Value));
}
/// <summary>
/// Rolling log file appender has no concept of cleaning up logs with a date stamp in their name.
/// This we have to clean them manually.
/// </summary>
/// <returns>A task.</returns>
private async ValueTask CleanLogs(string logFilePath)
{
Contract.RequiresNotNull(logFilePath);
const int threshold = 60;
int target = 50;
async ValueTask CleanFilesAsync()
{
var logsPath = Path.GetDirectoryName(logFilePath) ??
throw new InvalidOperationException("Could not resolve logs directory path: " + logFilePath);
var files = Directory.GetFiles(logsPath, LogPrefix + "*.txt");
if (files.Length > threshold)
{
var sorted = new SortedDictionary <DateTime, List <string> >();
foreach (var file in files)
{
var name = Path.GetFileName(file);
// assuming a format of log_20180717T130822Z.1.txt
int prefixLength = LogPrefix.Length;
var datePart = name[prefixLength..name.IndexOf(".", StringComparison.Ordinal)];
var success = DateTime.TryParseExact(
datePart,
"yyyyMMddTHHmmssZ",
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal,
out var date);
if (!success)
{
// the default date value will ensure value is added into sorted list
// and it will be deleted first!
LoggedConsole.WriteWarnLine($"Log file with name `{name}` has invalid date format and is being cleaned");
}
if (sorted.ContainsKey(date))
{
sorted[date].Add(file);
}
else
{
sorted.Add(date, new List <string>()
{
file
});
}
}
/// <summary>
/// Used to check that the keys in the indexProperties dictionary correspond to Properties in the SpectralIndexValues class.
/// Call this method before entering a loop because do not want the error message at every iteration through loop.
/// </summary>
public static void CheckExistenceOfSpectralIndexValues(Dictionary <string, IndexProperties> indexProperties)
{
foreach (var kvp in indexProperties)
{
if (!kvp.Value.IsSpectralIndex)
{
continue;
}
var success = CachedSelectors.ContainsKey(kvp.Key);
if (!success)
{
LoggedConsole.WriteWarnLine(
"### WARNING: The PROPERTY <" + kvp.Key + "> does not exist in the SpectralIndexValues class!");
}
}
}
/// <summary>
/// IMPORTANT: The indices passed in the dictionary "oneMinuteOfIndices" must be normalised.
/// </summary>
/// <param name="templates">The templates read from json file.</param>
/// <param name="templatesAsDictionary">The numerical part of each template.</param>
/// <param name="oneMinuteOfIndices">The normalised values of the indices derived from one minute of recording.</param>
/// <param name="minuteId">The minute ID, i.e. its temporal position.</param>
/// <returns>A single instance of a DescriptionResult.</returns>
public static DescriptionResult AnalyzeOneMinute(
FunctionalTemplate[] templates,
Dictionary <string, Dictionary <string, double[]> > templatesAsDictionary,
Dictionary <string, double[]> oneMinuteOfIndices,
int minuteId)
{
// initialise where the results will be stored.
var descriptionResult = new DescriptionResult(minuteId);
// now subject the indices to various content searches
foreach (var template in templates)
{
var algorithmType = template.Manifest.FeatureExtractionAlgorithm;
var templateIndices = templatesAsDictionary[template.Manifest.Name];
double score;
// Following line used where want to return a set of random scores for testing reasons.
//var score = new RandomNumber(DateTime.Now.Millisecond);
switch (algorithmType)
{
case 1:
score = ContentAlgorithms.GetFullBandContent1(oneMinuteOfIndices, template.Manifest, templateIndices);
break;
case 2:
score = ContentAlgorithms.GetBroadbandContent1(oneMinuteOfIndices, template.Manifest, templateIndices);
break;
case 3:
score = ContentAlgorithms.GetNarrowBandContent1(oneMinuteOfIndices, template.Manifest, templateIndices);
break;
default:
LoggedConsole.WriteWarnLine("Algorithm " + algorithmType + " does not exist.");
score = 0.0;
break;
}
var result = new KeyValuePair <string, double>(template.Manifest.Description, score);
descriptionResult.AddDescription(result);
}
return(descriptionResult);
}
/// <summary>
/// Used to check that the keys in the indexProperties dictionary correspond to Properties in the SpectralIndexValues class.
/// Call this method before entering a loop because do not want the error message at every iteration through loop.
/// </summary>
public static void CheckExistenceOfSpectralIndexValues(Dictionary <string, IndexProperties> indexProperties)
{
var siv = new SpectralIndexValues();
double[] dummyArray = null;
foreach (var kvp in indexProperties)
{
if (!kvp.Value.IsSpectralIndex)
{
continue;
}
var success = siv.TrySetPropertyValue(kvp.Key, dummyArray);
if (!success)
{
LoggedConsole.WriteWarnLine(
"### WARNING: The PROPERTY <" + kvp.Key + "> does not exist in the SpectralIndexValues class!");
}
}
}
public static Dictionary <string, double[, ]> GetSpectralIndexFilesAndConcatenate(
DirectoryInfo[] dirs,
string analysisType,
string[] keys,
IndexGenerationData indexGenerationData,
bool verbose = false)
{
TimeSpan indexCalcTimeSpan = indexGenerationData.IndexCalculationDuration;
Dictionary <string, double[, ]> spectrogramMatrices = new Dictionary <string, double[, ]>();
foreach (string key in keys)
{
//DateTime now1 = DateTime.Now;
string pattern = "*__" + analysisType + "." + key + ".csv";
var files = GetFilesInDirectories(dirs, pattern);
if (files.Length == 0)
{
LoggedConsole.WriteWarnLine($"{key} WARNING: No csv files found for KEY=" + key);
continue;
}
List <double[, ]> matrices = ConcatenateSpectralIndexFilesWithTimeCheck(files, indexCalcTimeSpan, key);
double[,] m = MatrixTools.ConcatenateMatrixRows(matrices);
//Dictionary<string, double[,]> dict = spectralIndexValues.ToTwoDimensionalArray(SpectralIndexValues.CachedSelectors, TwoDimensionalArray.Rotate90ClockWise);
m = MatrixTools.MatrixRotate90Anticlockwise(m);
spectrogramMatrices.Add(key, m);
//var now2 = DateTime.Now;
//var et = now2 - now1;
//if (verbose)
//{
// LoggedConsole.WriteLine($"\t\tTime to read <{key}> spectral index files = {et.TotalSeconds:f2} seconds");
//}
}
return(spectrogramMatrices);
}
/// <summary>
/// TODO TODO work on this method using website and javascript referred to by Anthony. (suncalc.net)
/// This method requires a properly formatted csv file containing sunrise/sunset data.
/// </summary>
/// <param name="width"></param>
/// <param name="dateTimeOffset"></param>
/// <param name="sunriseDatafile"></param>
/// <returns></returns>
public static Image <Rgb24> AddSunTrackToImage(int width, DateTimeOffset?dateTimeOffset, FileInfo sunriseDatafile)
{
// AT: I DON'T UNDERSTAND THIS CODE! I CAN'T FIX IT
LoggedConsole.WriteWarnLine("\t\tERROR: Sun track generation disabled - broken in long ago merge");
return(null);
// if (!sunriseDatafile.Exists)
// return null;
//<<<<<<< HEAD
// }
//
// if (siteName.StartsWith("Gympie") || siteName.StartsWith("Woondum3") || siteName.StartsWith("SERF"))
// {
// int dayOfYear = ((DateTimeOffset)dateTimeOffset).DayOfYear;
// double moonPhase = SunAndMoon.GetPhaseOfMoon((DateTimeOffset)dateTimeOffset);
// string strMoonPhase = SunAndMoon.ConvertMoonPhaseToString(moonPhase);
// throw new NotSupportedException("THE FOLLOWING FAILS IN PRODUCTION");
// Image<Rgb24> suntrack = SunAndMoon.AddSunTrackToImage(width, SunAndMoon.BrisbaneSunriseDatafile, dayOfYear, strMoonPhase);
// return suntrack;
// }
//
// return null;
// }
//
// public static Image<Rgb24> AddSunTrackToImage(int width, DateTimeOffset? dateTimeOffset, SiteDescription siteDescription)
// {
// return AddSunTrackToImage(width, dateTimeOffset, siteDescription.SiteName, siteDescription.Latitude, siteDescription.Longitude);
//=======
// int year = ((DateTimeOffset)dateTimeOffset).Year;
// int dayOfYear = ((DateTimeOffset)dateTimeOffset).DayOfYear;
// double moonPhase = SunAndMoon.GetPhaseOfMoon((DateTimeOffset)dateTimeOffset);
// string strMoonPhase = SunAndMoon.ConvertMoonPhaseToString(moonPhase);
// Image<Rgb24> suntrack = SunAndMoon.AddSunTrackToImage(width, sunriseDatafile, year, dayOfYear, strMoonPhase);
// return suntrack;
//>>>>>>> master
}
public void TestLogging()
{
var log = LogManager.GetLogger(typeof(Logging));
log.Prompt("Log test PROMPT");
log.Fatal("Log test FATAL");
log.Error("Log test ERROR");
log.Warn("Log test WARN");
log.Success("Log test SUCCESS");
log.Info("Log test INFO");
log.Debug("Log test DEBUG");
log.Trace("Log test TRACE");
log.Verbose("Log test VERBOSE");
LoggedConsole.WriteFatalLine("Clean wrapper FATAL", new Exception("I'm a fake"));
LoggedConsole.Log.Fatal("Clean log FATAL", new Exception("I'm a fake"));
LoggedConsole.WriteErrorLine("Clean wrapper ERROR");
LoggedConsole.Log.Error("Clean log ERROR");
LoggedConsole.WriteWarnLine("Clean wrapper WARN");
LoggedConsole.Log.Warn("Clean log WARN");
LoggedConsole.WriteSuccessLine("Clean wrapper SUCCESS");
LoggedConsole.Log.Success("Clean log SUCCESS");
LoggedConsole.WriteLine("Clean wrapper INFO");
LoggedConsole.Log.Info("Clean log INFO");
}
internal static void HangBeforeExit()
{
#if DEBUG
if (IsMsTestRunningMe)
{
return;
}
// if Michael is debugging with visual studio, this will prevent the window closing.
Process parentProcess = ProcessExtensions.ParentProcessUtilities.GetParentProcess();
if (Debugger.IsAttached)
{
if (parentProcess == null)
{
LoggedConsole.WriteWarnLine("WARNING: Unable to detect parent process, this is a windows specific tool.");
}
else if (parentProcess?.ProcessName == "devenv")
{
LoggedConsole.WriteSuccessLine("FINISHED: Press RETURN key to exit.");
Console.ReadLine();
}
}
#endif
}
/// <summary>
/// Converts summary indices to a tracks image
/// </summary>
/// <param name="listOfIndexProperties"></param>
/// <param name="dictionaryOfSummaryIndices"></param>
/// <param name="titleText"></param>
/// <param name="indexCalculationDuration"></param>
/// <param name="recordingStartDate"></param>
/// <param name="sunriseDataFile"></param>
/// <param name="errors"></param>
public static Bitmap DrawImageOfSummaryIndices(
Dictionary <string, IndexProperties> listOfIndexProperties,
Dictionary <string, double[]> dictionaryOfSummaryIndices,
string titleText,
TimeSpan indexCalculationDuration,
DateTimeOffset?recordingStartDate,
FileInfo sunriseDataFile = null,
List <GapsAndJoins> errors = null,
bool verbose = false)
{
// to translate past keys into current keys
Dictionary <string, string> translationDictionary = InitialiseIndexProperties.GetKeyTranslationDictionary();
const int trackHeight = DefaultTrackHeight;
int scaleLength = 0;
var bitmapList = new List <Tuple <IndexProperties, Image> >(dictionaryOfSummaryIndices.Keys.Count);
// accumulate the individual tracks in a List
foreach (string key in dictionaryOfSummaryIndices.Keys)
{
string correctKey = key;
if (!listOfIndexProperties.ContainsKey(key))
{
if (translationDictionary.ContainsKey(key))
{
correctKey = translationDictionary[key];
LoggedConsole.WriteWarnLine(
"The csv header is an unknown index <{0}>. Translated to <{1}>",
key,
correctKey);
}
else
{
if (verbose)
{
Logger.Warn(
"A index properties configuration could not be found for {0} (not even in the translation directory). Property is ignored and not rendered"
.Format2(key));
}
continue;
}
}
IndexProperties ip = listOfIndexProperties[correctKey];
if (!ip.DoDisplay)
{
continue;
}
string name = ip.Name;
double[] array = dictionaryOfSummaryIndices[key];
scaleLength = array.Length;
Image bitmap = ip.GetPlotImage(array, errors);
bitmapList.Add(Tuple.Create(ip, bitmap));
}
var listOfBitmaps = bitmapList
.OrderBy(tuple => tuple.Item1.Order)
.Select(tuple => tuple.Item2)
.Where(b => b != null).ToList();
//set up the composite image parameters
int X_offset = 2;
int graphWidth = X_offset + scaleLength;
int imageWidth = X_offset + scaleLength + TrackEndPanelWidth;
TimeSpan scaleDuration = TimeSpan.FromMinutes(scaleLength);
int imageHt = trackHeight * (listOfBitmaps.Count + 4); //+3 for title and top and bottom time tracks
Bitmap titleBmp = ImageTrack.DrawTitleTrack(imageWidth, trackHeight, titleText);
//Bitmap time1Bmp = ImageTrack.DrawTimeTrack(scaleDuration, TimeSpan.Zero, DrawSummaryIndices.TimeScale, graphWidth, TrackHeight, "Time (hours)");
TimeSpan xAxisPixelDuration = indexCalculationDuration;
TimeSpan fullDuration = TimeSpan.FromTicks(xAxisPixelDuration.Ticks * graphWidth);
Bitmap timeBmp1 = ImageTrack.DrawTimeRelativeTrack(fullDuration, graphWidth, trackHeight);
Bitmap timeBmp2 = timeBmp1;
Bitmap suntrack = null;
DateTimeOffset?dateTimeOffset = recordingStartDate;
if (dateTimeOffset.HasValue)
{
// draw extra time scale with absolute start time. AND THEN Do SOMETHING WITH IT.
timeBmp2 = ImageTrack.DrawTimeTrack(fullDuration, dateTimeOffset, graphWidth, trackHeight);
suntrack = SunAndMoon.AddSunTrackToImage(scaleLength, dateTimeOffset, sunriseDataFile);
}
//draw the composite bitmap
var imageList = new List <Image>();
imageList.Add(titleBmp);
imageList.Add(timeBmp1);
for (int i = 0; i < listOfBitmaps.Count; i++)
{
imageList.Add(listOfBitmaps[i]);
}
imageList.Add(timeBmp2);
imageList.Add(suntrack);
Bitmap compositeBmp = (Bitmap)ImageTools.CombineImagesVertically(imageList);
return(compositeBmp);
}
/// <summary>
/// All the passed files will be concatenated. Filtering needs to be done somewhere else.
/// </summary>
/// <param name="files">array of file names.</param>
/// <param name="indexCalcDuration">used to match rows of indices to elapsed time in file names.</param>
public static List <SummaryIndexValues> ConcatenateSummaryIndexFilesWithTimeCheck(FileInfo[] files, TimeSpan indexCalcDuration)
{
TimeSpan?offsetHint = new TimeSpan(10, 0, 0);
DateTimeOffset[] dtoArray = new DateTimeOffset[files.Length];
var summaryIndices = new List <SummaryIndexValues>();
// accumulate the start times for each of the files
for (int f = 0; f < files.Length; f++)
{
if (!files[f].Exists)
{
LoggedConsole.WriteWarnLine($"WARNING: Concatenation Time Check: MISSING FILE: {files[f].FullName}");
continue;
}
if (!FileDateHelpers.FileNameContainsDateTime(files[f].Name, out var date, offsetHint))
{
LoggedConsole.WriteWarnLine($"WARNING: Concatenation Time Check: INVALID DateTime in File Name {files[f].Name}");
}
dtoArray[f] = date;
}
// we use the fileName field to distinguish unique input source files
// this Set allows us to check they are unique and render joins
var sourceFileNames = new HashSet <string>();
// now loop through the files again to extract the indices
for (int i = 0; i < files.Length; i++)
{
if (!files[i].Exists)
{
continue;
}
var rowsOfCsvFile = Csv.ReadFromCsv <SummaryIndexValues>(files[i], throwOnMissingField: false);
// check all rows have fileName set
var thisSourceFileNames = new HashSet <string>();
foreach (var summaryIndexValues in rowsOfCsvFile)
{
if (summaryIndexValues.FileName.IsNullOrEmpty())
{
throw new InvalidOperationException($"A supplied summary index file did not have the `{nameof(SummaryIndexValues.FileName)}` field populated. File: {files[i].FullName}");
}
thisSourceFileNames.Add(summaryIndexValues.FileName);
}
// check all found filenames are unique
foreach (var sourceFileName in thisSourceFileNames)
{
if (sourceFileNames.Contains(sourceFileName))
{
throw new InvalidOperationException(
$"The summary index files already read previously contained the filename {sourceFileName} - duplicates are not allowed. File: {files[i].FullName}");
}
sourceFileNames.Add(sourceFileName);
}
summaryIndices.AddRange(rowsOfCsvFile);
// track the row counts
int partialRowCount = rowsOfCsvFile.Count();
// calculate elapsed time from the rows
int accumulatedRowMinutes = (int)Math.Round(partialRowCount * indexCalcDuration.TotalMinutes);
// calculate the partial elapsed minutes as indexed by file names.
var elapsedMinutesInFileNames = 0;
if (i < files.Length - 1)
{
TimeSpan elapsedTimeAccordingtoFileNames = dtoArray[i + 1] - dtoArray[i];
elapsedMinutesInFileNames = (int)Math.Round(elapsedTimeAccordingtoFileNames.TotalMinutes);
}
else
{
elapsedMinutesInFileNames = accumulatedRowMinutes; // a hack for the last file
}
// Check for Mismatch error in concatenation.
if (accumulatedRowMinutes != elapsedMinutesInFileNames)
{
string str1 = $"Concatenation: Elapsed Time Mismatch ERROR in csvFile {i + 1}/{files.Length}: {accumulatedRowMinutes} accumulatedRowMinutes != {elapsedMinutesInFileNames} elapsedMinutesInFileNames";
LoggedConsole.WriteWarnLine(str1);
//dictionary = RepairDictionaryOfArrays(dictionary, rowCounts[i], partialMinutes);
int scalingfactor = (int)Math.Round(60.0 / indexCalcDuration.TotalSeconds);
int minutesToAdd = elapsedMinutesInFileNames - accumulatedRowMinutes;
int rowsToAdd = minutesToAdd * scalingfactor;
// add in the missing summary index rows
for (int j = 0; j < rowsToAdd; j++)
{
var vector = new SummaryIndexValues {
FileName = MissingRowString
};
summaryIndices.Add(vector);
}
}
}
// Can prune the list of summary indices as required.
//int expectedRowCount = (int)Math.Round(numberOfMinutesInDay / indexCalcDuration.TotalMinutes);
//if (totalRowCount != expectedRowCount)
//{
// if (IndexMatrices.Verbose)
// LoggedConsole.WriteLine("WARNING: INCONSISTENT ELAPSED TIME CHECK from IndexMatrices.GetSummaryIndexFilesAndConcatenateWithTimeCheck() ");
// string str = String.Format(" Final Data Row Count = {0} Estimated Cumulative Duration = {1} minutes", totalRowCount, expectedRowCount);
// if (IndexMatrices.Verbose)
// LoggedConsole.WriteLine(str);
// dictionary = RepairDictionaryOfArrays(dictionary, totalRowCount, expectedRowCount);
//}
return(summaryIndices);
}
public static void Main(Arguments arguments)
{
var output = arguments.Output;
if (!output.Exists)
{
output.Create();
}
const string title = "# PRE-PROCESS SHORT AUDIO RECORDINGS FOR Convolutional DNN";
string date = "# DATE AND TIME: " + DateTime.Now;
LoggedConsole.WriteLine(title);
LoggedConsole.WriteLine(date);
LoggedConsole.WriteLine("# Input Query file: " + arguments.QueryWavFile);
LoggedConsole.WriteLine("# Input target file: " + arguments.TargtWavFile);
LoggedConsole.WriteLine("# Configure file: " + arguments.Config);
LoggedConsole.WriteLine("# Output directory: " + output.Name);
// 1. set up the necessary files
FileInfo queryWavfile = arguments.QueryWavFile.ToFileInfo();
FileInfo queryCsvfile = arguments.QueryCsvFile.ToFileInfo();
FileInfo targtWavfile = arguments.TargtWavFile.ToFileInfo();
FileInfo targtCsvfile = arguments.TargtCsvFile.ToFileInfo();
FileInfo configFile = arguments.Config.ToFileInfo();
DirectoryInfo opDir = output;
// 2. get the config dictionary
Config configuration = ConfigFile.Deserialize(configFile);
// below four lines are examples of retrieving info from Config config
//Config configuration = Yaml.Deserialise(configFile);
// string analysisIdentifier = configuration[AnalysisKeys.AnalysisName];
// int resampleRate = (int?)configuration[AnalysisKeys.ResampleRate] ?? AppConfigHelper.DefaultTargetSampleRate;
var configDict = new Dictionary <string, string>(configuration.ToDictionary());
configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
//bool makeSoxSonogram = (bool?)configuration[AnalysisKeys.MakeSoxSonogram] ?? false;
configDict[AnalysisKeys.AddTimeScale] = configuration[AnalysisKeys.AddTimeScale] ?? "true";
configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
// print out the parameters
LoggedConsole.WriteLine("\nPARAMETERS");
foreach (KeyValuePair <string, string> kvp in configDict)
{
LoggedConsole.WriteLine("{0} = {1}", kvp.Key, kvp.Value);
}
//set up the output file
//string header = "File Name, MinFreq(Hz), MaxFreq(Hz), StartTime(s), EndTime(s), Duration(s), Annotated by expert(Y-1/N-0),Correct Annotation(Y-1/N-0)";
string header = "File Name,MinFreq(Hz),MaxFreq(Hz),StartTime(s),EndTime(s),Duration(s),Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path2Spectrograms";
string opPath = Path.Combine(opDir.FullName, "OUTPUT.csv");
using (StreamWriter writer = new StreamWriter(opPath))
{
writer.WriteLine(header);
}
// reads the entire file
var data = FileTools.ReadTextFile(queryCsvfile.FullName);
// read single record from csv file
var record = CsvDataRecord.ReadDataRecord(data[1]);
if (!queryWavfile.Exists)
{
string warning = string.Format("FILE DOES NOT EXIST >>>," + arguments.QueryWavFile);
LoggedConsole.WriteWarnLine(warning);
return;
}
// ####################################################################
var result = AnalyseOneRecording(queryWavfile, configDict, record.EventStartSeconds, record.EventEndSeconds,
record.LowFrequencyHertz, record.HighFrequencyHertz, opDir);
// CONSTRUCT the outputline for csv file
// fileName,Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path
string line = string.Format("{0},{1},{2},{3:f2},{4:f2},{5:f2},{6:f1},{7:f3},{8:f3},{9:f3},{10}",
record.WavFileName, record.LowFrequencyHertz, record.HighFrequencyHertz,
record.EventStartSeconds.TotalSeconds, record.EventEndSeconds.TotalSeconds,
result.SnrStatistics.ExtractDuration.TotalSeconds,
result.SnrStatistics.Threshold, result.SnrStatistics.Snr,
result.SnrStatistics.FractionOfFramesExceedingThreshold, result.SnrStatistics.FractionOfFramesExceedingOneThirdSnr,
result.SpectrogramFile.FullName);
// It is helpful to write to the output file as we go, so as to keep a record of where we are up to.
// This requires to open and close the output file at each iteration
using (StreamWriter writer = new StreamWriter(opPath, true))
{
writer.WriteLine(line);
}
// ####################################################################
result = AnalyseOneRecording(targtWavfile, configDict, record.EventStartSeconds, record.EventEndSeconds,
record.LowFrequencyHertz, record.HighFrequencyHertz, opDir);
// CONSTRUCT the outputline for csv file
// fileName,Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path
//string line = String.Format("{0},{1},{2},{3:f2},{4:f2},{5:f2},{6:f1},{7:f3},{8:f3},{9:f3},{10}",
// record.wavFile_name, record.low_frequency_hertz, record.high_frequency_hertz,
// record.event_start_seconds.TotalSeconds, record.event_end_seconds.TotalSeconds,
// result.SnrStatistics.ExtractDuration.TotalSeconds,
// result.SnrStatistics.Threshold, result.SnrStatistics.Snr,
// result.SnrStatistics.FractionOfFramesExceedingThreshold, result.SnrStatistics.FractionOfFramesExceedingOneThirdSNR,
// result.SpectrogramFile.FullName);
// It is helpful to write to the output file as we go, so as to keep a record of where we are up to.
// This requires to open and close the output file at each iteration
using (StreamWriter writer = new StreamWriter(opPath, true))
{
writer.WriteLine(line);
}
} // end MAIN()
/// <summary>
/// This test checks two text/csv files to determine if they are the same.
/// </summary>
public static void FileEqualityTest(string testName, FileInfo testFile, FileInfo benchmarkFile)
{
LoggedConsole.WriteLine("# FILE EQUALITY TEST: Starting benchmark test for " + testName + ":");
LoggedConsole.WriteLine("# Comparing file <" + testFile.Name + "> with <" + benchmarkFile.Name + ">");
if (!testFile.Exists)
{
LoggedConsole.WriteWarnLine(" " + testName + " Test File <" + testFile.Name + "> does not exist.");
return;
}
if (!benchmarkFile.Exists)
{
LoggedConsole.WriteWarnLine(" " + testName + ": the Benchmark File <" + benchmarkFile.Name + "> does not exist.");
// check that the benchmark directory exists - if not create it.
var benchmarkDir = benchmarkFile.Directory;
if (!benchmarkDir.Exists)
{
LoggedConsole.WriteWarnLine(" Creating Benchmark Directory");
benchmarkDir.Create();
}
LoggedConsole.WriteWarnLine(" Writing the Test File as a future Benchmark File");
File.Copy(testFile.FullName, benchmarkFile.FullName, false);
return;
}
var lines1 = FileTools.ReadTextFile(testFile.FullName);
var lines2 = FileTools.ReadTextFile(benchmarkFile.FullName);
if (lines1.Count == 0)
{
LoggedConsole.WriteWarnLine(" " + testName + " File1 contains zero lines.");
return;
}
if (lines2.Count == 0)
{
LoggedConsole.WriteWarnLine(" " + testName + " File2 contains zero lines.");
return;
}
if (lines1.Count != lines2.Count)
{
LoggedConsole.WriteWarnLine(" " + testName + " The two files do not contain the same number of lines.");
LoggedConsole.WriteWarnLine(" line count 1 <" + lines1.Count + "> != line count 2 <" + lines2.Count + ">");
return;
}
var allOk = true;
for (int i = 0; i < lines2.Count; i++)
{
if (!lines1[i].Equals(lines2[i]))
{
LoggedConsole.WriteWarnLine($"Line {i}: <{lines1[i]}> != benchmark <{lines2[i]}>");
allOk = false;
}
}
if (allOk)
{
LoggedConsole.WriteSuccessLine("# SUCCESS! Passed the FILE EQUALITY TEST.");
}
else
{
LoggedConsole.WriteWarnLine("# FAILED TEST! FILES ARE NOT THE SAME!");
}
LoggedConsole.WriteLine("# Completed benchmark test.");
}
/// <summary>
/// Rolling log file appender has no concept of cleaning up logs with a date stamp in their name.
/// This we have to clean them manually.
/// </summary>
/// <returns>A task.</returns>
private async Task CleanLogs(string logFilePath)
{
Contract.RequiresNotNull(logFilePath);
const int threshold = 60;
int target = 50;
void CleanFiles()
{
var logsPath = Path.GetDirectoryName(logFilePath) ??
throw new InvalidOperationException("Could not resolve logs directory path: " + logFilePath);
var files = Directory.GetFiles(logsPath, LogPrefix + "*.txt");
if (files.Length > threshold)
{
var sorted = new SortedDictionary <DateTime, List <string> >();
foreach (var file in files)
{
var name = Path.GetFileName(file);
// assuming a format of log_20180717T130822Z.1.txt
int prefixLength = LogPrefix.Length;
var datePart = name.Substring(prefixLength, name.IndexOf(".", StringComparison.Ordinal) - prefixLength);
var success = DateTime.TryParseExact(
datePart,
"yyyyMMddTHHmmssZ",
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal,
out var date);
if (!success)
{
// the default date value will ensure value is added into sorted list
// and it will be deleted first!
LoggedConsole.WriteWarnLine($"Log file with name `{name}` has invalid date format and is being cleaned");
}
if (sorted.ContainsKey(date))
{
sorted[date].Add(file);
}
else
{
sorted.Add(date, new List <string>()
{
file
});
}
}
// then delete the last 10 or so (this way we batch deletes)
var toDelete = files.Length - target;
foreach (var kvp in sorted)
{
foreach (var file in kvp.Value)
{
File.Delete(file);
toDelete--;
if (toDelete <= 0)
{
return;
}
}
}
}
}
try
{
// ReSharper disable once RedundantCast
await Task.Run((Action)CleanFiles);
}
catch (Exception ex)
{
LoggedConsole.WriteFatalLine("Log cleaning failed, this is a bug, please report it.", ex);
}
}
/// <summary>
/// 2. Analyses long audio recording (mp3 or wav) as per passed config file. Outputs an events.csv file AND an
/// indices.csv file
/// Signed off: Michael Towsey 4th December 2012
/// </summary>
public static void Execute(Arguments arguments)
{
if (arguments == null)
{
throw new NoDeveloperMethodException();
}
LoggedConsole.WriteLine("# PROCESS LONG RECORDING");
LoggedConsole.WriteLine("# DATE AND TIME: " + DateTime.Now);
// 1. set up the necessary files
var sourceAudio = arguments.Source;
var configFile = arguments.Config.ToFileInfo();
var outputDirectory = arguments.Output;
var tempFilesDirectory = arguments.TempDir;
// if a temp dir is not given, use output dir as temp dir
if (tempFilesDirectory == null)
{
Log.Warn("No temporary directory provided, using output directory");
tempFilesDirectory = outputDirectory;
}
// try an automatically find the config file
if (configFile == null)
{
throw new FileNotFoundException("No config file argument provided");
}
else if (!configFile.Exists)
{
Log.Warn($"Config file {configFile.FullName} not found... attempting to resolve config file");
// we use .ToString() here to get the original input string - Using fullname always produces an absolute path wrt to pwd... we don't want to prematurely make asusmptions:
// e.g. We require a missing absolute path to fail... that wouldn't work with .Name
// e.g. We require a relative path to try and resolve, using .FullName would fail the first absolute check inside ResolveConfigFile
configFile = ConfigFile.Resolve(configFile.ToString(), Directory.GetCurrentDirectory().ToDirectoryInfo());
}
if (arguments.StartOffset.HasValue ^ arguments.EndOffset.HasValue)
{
throw new InvalidStartOrEndException("If StartOffset or EndOffset is specified, then both must be specified");
}
if (arguments.StartOffset.HasValue && arguments.EndOffset.HasValue && arguments.EndOffset.Value <= arguments.StartOffset.Value)
{
throw new InvalidStartOrEndException("Start offset must be less than end offset.");
}
LoggedConsole.WriteLine("# Recording file: " + sourceAudio.FullName);
LoggedConsole.WriteLine("# Configuration file: " + configFile);
LoggedConsole.WriteLine("# Output folder: " + outputDirectory);
LoggedConsole.WriteLine("# Temp File Directory: " + tempFilesDirectory);
// optionally copy logs / config to make results easier to understand
// TODO: remove, see https://github.com/QutEcoacoustics/audio-analysis/issues/133
if (arguments.WhenExitCopyConfig || arguments.WhenExitCopyLog)
{
AppDomain.CurrentDomain.ProcessExit += (sender, args) => { Cleanup(arguments, configFile); };
}
// 2. initialize the analyzer
// we're changing the way resolving config files works. Ideally, we'd like to use statically typed config files
// but we can't do that unless we know which type we have to load first! Currently analyzer to load is in
// the config file so we can't know which analyzer we can use. Thus we will change to using the file name,
// or an argument to resolve the analyzer to load.
// Get analysis name:
IAnalyser2 analyzer = FindAndCheckAnalyzer <IAnalyser2>(arguments.AnalysisIdentifier, configFile.Name);
// 2. get the analysis config
AnalyzerConfig configuration = analyzer.ParseConfig(configFile);
SaveBehavior saveIntermediateWavFiles = configuration.SaveIntermediateWavFiles;
bool saveIntermediateDataFiles = configuration.SaveIntermediateCsvFiles;
SaveBehavior saveSonogramsImages = configuration.SaveSonogramImages;
bool filenameDate = configuration.RequireDateInFilename;
if (configuration[AnalysisKeys.AnalysisName].IsNotWhitespace())
{
Log.Warn("Your config file has `AnalysisName` set - this property is deprecated and ignored");
}
// AT 2018-02: changed logic so default index properties loaded if not provided
FileInfo indicesPropertiesConfig = IndexProperties.Find(configuration, configFile);
if (indicesPropertiesConfig == null || !indicesPropertiesConfig.Exists)
{
Log.Warn("IndexProperties config can not be found! Loading a default");
indicesPropertiesConfig = ConfigFile.Default <Dictionary <string, IndexProperties> >();
}
LoggedConsole.WriteLine("# IndexProperties Cfg: " + indicesPropertiesConfig.FullName);
// min score for an acceptable event
Log.Info("Minimum event threshold has been set to " + configuration.EventThreshold);
FileSegment.FileDateBehavior defaultBehavior = FileSegment.FileDateBehavior.Try;
if (filenameDate)
{
if (!FileDateHelpers.FileNameContainsDateTime(sourceAudio.Name))
{
throw new InvalidFileDateException(
"When RequireDateInFilename option is set, the filename of the source audio file must contain "
+ "a valid AND UNAMBIGUOUS date. Such a date was not able to be parsed.");
}
defaultBehavior = FileSegment.FileDateBehavior.Required;
}
// 3. initilize AnalysisCoordinator class that will do the analysis
var analysisCoordinator = new AnalysisCoordinator(
new LocalSourcePreparer(),
saveIntermediateWavFiles,
false,
arguments.Parallel);
// 4. get the segment of audio to be analysed
// if tiling output, specify that FileSegment needs to be able to read the date
var fileSegment = new FileSegment(sourceAudio, arguments.AlignToMinute, null, defaultBehavior);
var bothOffsetsProvided = arguments.StartOffset.HasValue && arguments.EndOffset.HasValue;
if (bothOffsetsProvided)
{
fileSegment.SegmentStartOffset = TimeSpan.FromSeconds(arguments.StartOffset.Value);
fileSegment.SegmentEndOffset = TimeSpan.FromSeconds(arguments.EndOffset.Value);
}
else
{
Log.Debug("Neither start nor end segment offsets provided. Therefore both were ignored.");
}
// 6. initialize the analysis settings object
var analysisSettings = analyzer.DefaultSettings;
analysisSettings.ConfigFile = configFile;
analysisSettings.Configuration = configuration;
analysisSettings.AnalysisOutputDirectory = outputDirectory;
analysisSettings.AnalysisTempDirectory = tempFilesDirectory;
analysisSettings.AnalysisDataSaveBehavior = saveIntermediateDataFiles;
analysisSettings.AnalysisImageSaveBehavior = saveSonogramsImages;
analysisSettings.AnalysisChannelSelection = arguments.Channels;
analysisSettings.AnalysisMixDownToMono = arguments.MixDownToMono;
var segmentDuration = configuration.SegmentDuration?.Seconds();
if (!segmentDuration.HasValue)
{
segmentDuration = analysisSettings.AnalysisMaxSegmentDuration ?? TimeSpan.FromMinutes(1);
Log.Warn(
$"Can't read `{nameof(AnalyzerConfig.SegmentDuration)}` from config file. "
+ $"Default value of {segmentDuration} used)");
}
analysisSettings.AnalysisMaxSegmentDuration = segmentDuration.Value;
var segmentOverlap = configuration.SegmentOverlap?.Seconds();
if (!segmentOverlap.HasValue)
{
segmentOverlap = analysisSettings.SegmentOverlapDuration;
Log.Warn(
$"Can't read `{nameof(AnalyzerConfig.SegmentOverlap)}` from config file. "
+ $"Default value of {segmentOverlap} used)");
}
analysisSettings.SegmentOverlapDuration = segmentOverlap.Value;
// set target sample rate
var resampleRate = configuration.ResampleRate;
if (!resampleRate.HasValue)
{
resampleRate = analysisSettings.AnalysisTargetSampleRate ?? AppConfigHelper.DefaultTargetSampleRate;
Log.Warn(
$"Can't read {nameof(configuration.ResampleRate)} from config file. "
+ $"Default value of {resampleRate} used)");
}
analysisSettings.AnalysisTargetSampleRate = resampleRate;
Log.Info(
$"{nameof(configuration.SegmentDuration)}={segmentDuration}, "
+ $"{nameof(configuration.SegmentOverlap)}={segmentOverlap}, "
+ $"{nameof(configuration.ResampleRate)}={resampleRate}");
// 7. ####################################### DO THE ANALYSIS ###################################
LoggedConsole.WriteLine("START ANALYSIS ...");
var analyserResults = analysisCoordinator.Run(fileSegment, analyzer, analysisSettings);
// ##############################################################################################
// 8. PROCESS THE RESULTS
LoggedConsole.WriteLine(string.Empty);
LoggedConsole.WriteLine("START PROCESSING RESULTS ...");
if (analyserResults == null)
{
LoggedConsole.WriteErrorLine("###################################################\n");
LoggedConsole.WriteErrorLine("The Analysis Run Coordinator has returned a null result.");
LoggedConsole.WriteErrorLine("###################################################\n");
throw new AnalysisOptionDevilException();
}
// Merge and correct main result types
EventBase[] mergedEventResults = ResultsTools.MergeResults(analyserResults, ar => ar.Events, ResultsTools.CorrectEvent);
SummaryIndexBase[] mergedIndicesResults = ResultsTools.MergeResults(analyserResults, ar => ar.SummaryIndices, ResultsTools.CorrectSummaryIndex);
SpectralIndexBase[] mergedSpectralIndexResults = ResultsTools.MergeResults(analyserResults, ar => ar.SpectralIndices, ResultsTools.CorrectSpectrumIndex);
// not an exceptional state, do not throw exception
if (mergedEventResults != null && mergedEventResults.Length == 0)
{
LoggedConsole.WriteWarnLine("The analysis produced no EVENTS (mergedResults had zero count)");
}
if (mergedIndicesResults != null && mergedIndicesResults.Length == 0)
{
LoggedConsole.WriteWarnLine("The analysis produced no Summary INDICES (mergedResults had zero count)");
}
if (mergedSpectralIndexResults != null && mergedSpectralIndexResults.Length == 0)
{
LoggedConsole.WriteWarnLine("The analysis produced no Spectral INDICES (merged results had zero count)");
}
// 9. CREATE SUMMARY INDICES IF NECESSARY (FROM EVENTS)
#if DEBUG
// get the duration of the original source audio file - need this to convert Events datatable to Indices Datatable
var audioUtility = new MasterAudioUtility(tempFilesDirectory);
var mimeType = MediaTypes.GetMediaType(sourceAudio.Extension);
var sourceInfo = audioUtility.Info(sourceAudio);
// updated by reference all the way down in LocalSourcePreparer
Debug.Assert(fileSegment.TargetFileDuration == sourceInfo.Duration);
#endif
var duration = fileSegment.TargetFileDuration.Value;
ResultsTools.ConvertEventsToIndices(
analyzer,
mergedEventResults,
ref mergedIndicesResults,
duration,
configuration.EventThreshold);
int eventsCount = mergedEventResults?.Length ?? 0;
int numberOfRowsOfIndices = mergedIndicesResults?.Length ?? 0;
// 10. Allow analysers to post-process
// TODO: remove results directory if possible
var instanceOutputDirectory =
AnalysisCoordinator.GetNamedDirectory(analysisSettings.AnalysisOutputDirectory, analyzer);
// 11. IMPORTANT - this is where IAnalyser2's post processor gets called.
// Produces all spectrograms and images of SPECTRAL INDICES.
// Long duration spectrograms are drawn IFF analysis type is Towsey.Acoustic
analyzer.SummariseResults(analysisSettings, fileSegment, mergedEventResults, mergedIndicesResults, mergedSpectralIndexResults, analyserResults);
// 12. SAVE THE RESULTS
string fileNameBase = Path.GetFileNameWithoutExtension(sourceAudio.Name);
var eventsFile = ResultsTools.SaveEvents(analyzer, fileNameBase, instanceOutputDirectory, mergedEventResults);
var indicesFile = ResultsTools.SaveSummaryIndices(analyzer, fileNameBase, instanceOutputDirectory, mergedIndicesResults);
var spectraFile = ResultsTools.SaveSpectralIndices(analyzer, fileNameBase, instanceOutputDirectory, mergedSpectralIndexResults);
// 13. THIS IS WHERE SUMMARY INDICES ARE PROCESSED
// Convert summary indices to black and white tracks image
if (mergedIndicesResults == null)
{
Log.Info("No summary indices produced");
}
else
{
if (indicesPropertiesConfig == null || !indicesPropertiesConfig.Exists)
{
throw new InvalidOperationException("Cannot process indices without an index configuration file, the file could not be found!");
}
// this arbitrary amount of data.
if (mergedIndicesResults.Length > 5000)
{
Log.Warn("Summary Indices Image not able to be drawn - there are too many indices to render");
}
else
{
var basename = Path.GetFileNameWithoutExtension(fileNameBase);
string imageTitle = $"SOURCE:{basename}, {Meta.OrganizationTag}; ";
// Draw Tracks-Image of Summary indices
// set time scale resolution for drawing of summary index tracks
TimeSpan timeScale = TimeSpan.FromSeconds(0.1);
Bitmap tracksImage =
IndexDisplay.DrawImageOfSummaryIndices(
IndexProperties.GetIndexProperties(indicesPropertiesConfig),
indicesFile,
imageTitle,
timeScale,
fileSegment.TargetFileStartDate);
var imagePath = FilenameHelpers.AnalysisResultPath(instanceOutputDirectory, basename, "SummaryIndices", ImageFileExt);
tracksImage.Save(imagePath);
}
}
// 14. wrap up, write stats
LoggedConsole.WriteLine("INDICES CSV file(s) = " + (indicesFile?.Name ?? "<<No indices result, no file!>>"));
LoggedConsole.WriteLine("\tNumber of rows (i.e. minutes) in CSV file of indices = " + numberOfRowsOfIndices);
LoggedConsole.WriteLine(string.Empty);
if (eventsFile == null)
{
LoggedConsole.WriteLine("An Events CSV file was NOT returned.");
}
else
{
LoggedConsole.WriteLine("EVENTS CSV file(s) = " + eventsFile.Name);
LoggedConsole.WriteLine("\tNumber of events = " + eventsCount);
}
Log.Success($"Analysis Complete.\nSource={sourceAudio.Name}\nOutput={instanceOutputDirectory.FullName}");
}
/// <summary>
/// HERVE GLOTIN
/// Combined audio2csv + zooming spectrogram task.
/// This is used to analyse Herve Glotin's BIRD50 data set.
/// ############################# IMPORTANT ########################################
/// In order to analyse the short recordings in BIRD50 dataset, need following change to code:
/// need to modify AudioAnalysis.AnalysisPrograms.AcousticIndices.cs #line648
/// need to change AnalysisMinSegmentDuration = TimeSpan.FromSeconds(20),
/// to AnalysisMinSegmentDuration = TimeSpan.FromSeconds(1),
/// THIS iS to analyse BIRD50 short recordings.
/// </summary>
public static void HiRes1()
{
string recordingPath = @"C:\SensorNetworks\WavFiles\TestRecordings\TEST_7min_artificial.wav";
//// HERVE GLOTIN BIRD50 TRAINING RECORDINGS
//DirectoryInfo dataDir = new DirectoryInfo(@"D:\SensorNetworks\WavFiles\Glotin\Bird50\AmazonBird50_training_input");
//string parentDir = @"C:\SensorNetworks\Output\BIRD50";
//string speciesLabelsFile = parentDir + @"\AmazonBird50_training_output.csv";
//int speciesCount = 50;
//////set file name format -depends on train or test. E.g. "ID0003";
//string fileStemFormatString = "ID{0:d4}"; // for training files
//string indexPropertiesConfig = @"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\IndexPropertiesConfigHiRes.yml";
//string learningMode = "Train";
//// HERVE GLOTIN BIRD50 TESTING RECORDINGS
DirectoryInfo dataDir = new DirectoryInfo(@"D:\SensorNetworks\WavFiles\Glotin\Bird50\AmazonBird50_testing_input");
string parentDir = @"C:\SensorNetworks\Output\BIRD50";
string speciesLabelsFile = null;
int speciesCount = 50;
////set file name format -depends on train or test. E.g. "ID0003";
string fileStemFormatString = "ID1{0:d3}"; // for testing files
string indexPropertiesConfig = @"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\IndexPropertiesConfigHiRes.yml";
string learningMode = "Test";
// HERVE GLOTIN BOMBYX WHALE RECORDINGS
//DirectoryInfo dataDir = new DirectoryInfo(@"C:\SensorNetworks\WavFiles\WhaleFromGlotin");
//string parentDir = @"C:\SensorNetworks\Output\Glotin\Bombyx_SpermWhales";
//string speciesLabelsFile = null;
//int speciesCount = 0;
//////set file name format -depends on train or test. E.g. "ID0003";
//string fileStemFormatString = null;
////string fileStemFormatString = "ID1{0:d3}"; // for testing files
//string indexPropertiesConfig = @"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\IndexPropertiesConfigHiResGianniPavan.yml";
//string learningMode = "Train";
// GIANNI PAVAN SASSAFRAS RECORDINGS
//DirectoryInfo dataDir = new DirectoryInfo(@"C:\SensorNetworks\WavFiles\GianniPavan\SABIOD - TEST SASSOFRATINO");
//string parentDir = @"C:\SensorNetworks\Output\GianniPavan";
//string speciesLabelsFile = null;
//int speciesCount = 0;
//string fileStemFormatString = null;
//string indexPropertiesConfig = @"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\IndexPropertiesConfigHiResGianniPavan.yml";
//string learningMode = "Train";
// ######################################################################
string outputDir = parentDir + @"\" + learningMode;
string imageOutputDir = parentDir + @"\" + learningMode + "Images";
string csvDir = outputDir + @"\Towsey.Acoustic";
string zoomOutputDir = outputDir;
string audio2csvConfigPath = @"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\Towsey.AcousticHiRes.yml";
string hiResZoomConfigPath = @"C:\Work\GitHub\audio-analysis\AudioAnalysis\AnalysisConfigFiles\SpectrogramHiResConfig.yml";
FileInfo[] wavFiles = { new FileInfo(recordingPath) };
// comment next two lines when debugging a single recording file
string match = @"*.wav";
wavFiles = dataDir.GetFiles(match, SearchOption.AllDirectories);
// READ IN THE SPECIES LABELS FILE AND SET UP THE DATA
string[] fileID = new string[wavFiles.Length];
int[] speciesID = new int[speciesCount];
if (speciesLabelsFile != null)
{
BirdClefExperiment1.ReadGlotinsSpeciesLabelFile(speciesLabelsFile, wavFiles.Length, out fileID, out speciesID);
}
else // make seperate species name for each file
{
speciesID = new int[wavFiles.Length];
}
//LOOP THROUGH ALL WAV FILES
//for (int i = 538; i < 539; i++)
//for (int i = 0; i < 8; i++)
for (int i = 0; i < wavFiles.Length; i++)
{
FileInfo file = wavFiles[i];
recordingPath = file.FullName;
string idName = Path.GetFileNameWithoutExtension(file.FullName);
string name = string.Format("{0}_Species{1:d2}", idName, speciesID[i]);
outputDir = parentDir + @"\" + learningMode + @"\" + name;
csvDir = parentDir + @"\" + learningMode + @"\" + name + @"\Towsey.Acoustic";
zoomOutputDir = outputDir;
Console.WriteLine("\n\n");
Console.WriteLine($@">>>>{i}: File<{name}>");
try
{
// A: analyse the recording files == audio2csv.
var audio2csvArguments = new AnalyseLongRecordings.AnalyseLongRecording.Arguments
{
Source = recordingPath.ToFileInfo(),
Config = audio2csvConfigPath,
Output = outputDir.ToDirectoryInfo(),
};
if (!audio2csvArguments.Source.Exists)
{
LoggedConsole.WriteWarnLine(" >>>>>>>>>>>> WARNING! The Source Recording file cannot be found! This will cause an exception.");
}
if (!File.Exists(audio2csvArguments.Config))
{
LoggedConsole.WriteWarnLine(" >>>>>>>>>>>> WARNING! The Configuration file cannot be found! This will cause an exception.");
}
AnalyseLongRecordings.AnalyseLongRecording.Execute(audio2csvArguments);
// B: Concatenate the summary indices and produce images
// Use the Zoomingspectrograms action.
// need to find out how long the recording is.
string fileName = audio2csvArguments.Source.BaseName();
string testFileName = fileName + @"__Towsey.Acoustic.ACI.csv";
List <string> data = FileTools.ReadTextFile(Path.Combine(csvDir, testFileName));
int lineCount = data.Count - 1; // -1 for header.
int imageWidth = lineCount;
//assume scale is index calculation duration = 0.1s
// i.e. image resolution 0.1s/px. or 600px/min
double focalMinute = (double)lineCount / 600 / 2;
if (focalMinute < 0.016666)
{
focalMinute = 0.016666; // shortest recording = 1 second.
}
var zoomingArguments = new DrawZoomingSpectrograms.Arguments
{
// use the default set of index properties in the AnalysisConfig directory.
SourceDirectory = csvDir,
Output = zoomOutputDir,
SpectrogramZoomingConfig = hiResZoomConfigPath,
// draw a focused multi-resolution pyramid of images
ZoomAction = DrawZoomingSpectrograms.Arguments.ZoomActionType.Focused,
};
LoggedConsole.WriteLine("# Spectrogram Zooming config : " + zoomingArguments.SpectrogramZoomingConfig);
LoggedConsole.WriteLine("# Input Directory : " + zoomingArguments.SourceDirectory);
LoggedConsole.WriteLine("# Output Directory : " + zoomingArguments.Output);
var common = new ZoomParameters(zoomingArguments.SourceDirectory.ToDirectoryEntry(), zoomingArguments.SpectrogramZoomingConfig.ToFileEntry(), false);
// Create directory if not exists
if (!Directory.Exists(zoomingArguments.Output))
{
Directory.CreateDirectory(zoomingArguments.Output);
}
ZoomFocusedSpectrograms.DrawStackOfZoomedSpectrograms(
zoomingArguments.SourceDirectory.ToDirectoryInfo(),
zoomingArguments.Output.ToDirectoryInfo(),
common,
TimeSpan.FromMinutes(focalMinute),
imageWidth,
AcousticIndices.TowseyAcoustic);
// DRAW THE VARIOUS IMAGES
string fileStem = fileName;
if (fileStemFormatString != null)
{
fileStem = string.Format(fileStemFormatString, i + 1); // training images
}
var ldfcSpectrogramArguments = new DrawLongDurationSpectrograms.Arguments
{
// use the default set of index properties in the AnalysisConfig directory.
InputDataDirectory = csvDir,
OutputDirectory = imageOutputDir,
IndexPropertiesConfig = indexPropertiesConfig,
};
// there are two possible tasks
// 1: draw the aggregated grey scale spectrograms
int secDuration = DrawLongDurationSpectrograms.DrawAggregatedSpectrograms(ldfcSpectrogramArguments, fileStem);
// 2: draw the coloured ridge spectrograms
secDuration = DrawLongDurationSpectrograms.DrawRidgeSpectrograms(ldfcSpectrogramArguments, fileStem);
// copy files
// POW, EVN, SPT, RHZ, RVT, RPS, RNG
string[] copyArray = { "POW", "EVN", "SPT", "RHZ", "RVT", "RPS", "RNG" };
DirectoryInfo sourceDirectory = new DirectoryInfo(csvDir);
string destinationDirectory = parentDir + @"\TrainingClassifier";
foreach (string key in copyArray)
{
// ID0002__Towsey.Acoustic.BGN.csv fileName += @"__Towsey.Acoustic.ACI.csv";
string sourceFileName = string.Format(idName + "__Towsey.Acoustic." + key + ".csv");
string sourcePath = Path.Combine(sourceDirectory.FullName, sourceFileName);
string nameOfParentDirectory = sourceDirectory.Parent.Name;
string destinationFileName = string.Format(nameOfParentDirectory + "." + key + ".csv");
string destinationPath = Path.Combine(destinationDirectory, destinationFileName);
File.Copy(sourcePath, destinationPath, true);
}
} // try block
catch (Exception e)
{
LoggedConsole.WriteErrorLine(string.Format("ERROR!!!!! RECORDING {0} FILE {1}", i, name));
LoggedConsole.WriteErrorLine(string.Format(e.ToString()));
}
} // end loop through all wav files
} // HiRes1()
/// <summary>
/// Currently this method is called by only one species recognizer - LitoriaCaerulea.
/// </summary>
/// <param name="ipArray">an array of decibel values.</param>
/// <param name="framesPerSecond">the frame rate.</param>
/// <param name="decibelThreshold">Ignore frames below this threshold.</param>
/// <param name="dctDuration">Duration in seconds of the required DCT.</param>
/// <param name="minOscFreq">minimum oscillation frequency.</param>
/// <param name="maxOscFreq">maximum oscillation frequency.</param>
/// <param name="dctThreshold">Threshold for the maximum DCT coefficient.</param>
/// <param name="dctScores">an array of dct scores.</param>
/// <param name="oscFreq">an array of oscillation frequencies.</param>
public static void DetectOscillations(
double[] ipArray,
double framesPerSecond,
double decibelThreshold,
double dctDuration,
double minOscFreq,
double maxOscFreq,
double dctThreshold,
out double[] dctScores,
out double[] oscFreq)
{
int dctLength = (int)Math.Round(framesPerSecond * dctDuration);
int minIndex = (int)(minOscFreq * dctDuration * 2); //multiply by 2 because index = Pi and not 2Pi
int maxIndex = (int)(maxOscFreq * dctDuration * 2); //multiply by 2 because index = Pi and not 2Pi
if (maxIndex > dctLength)
{
LoggedConsole.WriteWarnLine("MaxIndex > DCT length. Therefore set maxIndex = DCT length.");
maxIndex = dctLength;
}
int length = ipArray.Length;
dctScores = new double[length];
oscFreq = new double[length];
//set up the cosine coefficients
double[,] cosines = MFCCStuff.Cosines(dctLength, dctLength);
//following two lines write bmp image of cosine matrix values for checking.
//string bmpPath = @"C:\SensorNetworks\Output\cosines.png";
//ImageTools.DrawMatrix(cosines, bmpPath, true);
for (int r = 1; r < length - dctLength; r++)
{
// only stop if current location is a peak
if (ipArray[r] < ipArray[r - 1] || ipArray[r] < ipArray[r + 1])
{
continue;
}
// only stop if current location is a peak
if (ipArray[r] < decibelThreshold)
{
continue;
}
// extract array and ready for DCT
var dctArray = DataTools.Subarray(ipArray, r, dctLength);
dctArray = DataTools.SubtractMean(dctArray);
double[] dctCoefficient = MFCCStuff.DCT(dctArray, cosines);
// convert to absolute values because not interested in negative values due to phase.
for (int i = 0; i < dctLength; i++)
{
dctCoefficient[i] = Math.Abs(dctCoefficient[i]);
}
// remove low freq oscillations from consideration
int thresholdIndex = minIndex / 4;
for (int i = 0; i < thresholdIndex; i++)
{
dctCoefficient[i] = 0.0;
}
dctCoefficient = DataTools.normalise2UnitLength(dctCoefficient);
int indexOfMaxValue = DataTools.GetMaxIndex(dctCoefficient);
//mark DCT location with oscillation freq, only if oscillation freq is in correct range and amplitude
if (indexOfMaxValue >= minIndex && indexOfMaxValue <= maxIndex && dctCoefficient[indexOfMaxValue] > dctThreshold)
{
for (int i = 0; i < dctLength; i++)
{
if (dctScores[r + i] < dctCoefficient[indexOfMaxValue])
{
dctScores[r + i] = dctCoefficient[indexOfMaxValue];
oscFreq[r + i] = indexOfMaxValue / dctDuration / 2;
}
}
}
}
}