// end MAIN()
public static AudioToSonogramResult AnalyseOneRecording(CsvDataRecord dataRecord, Dictionary <string, string> configDict, DirectoryInfo opDir)
{
// ############## IMPORTANT PARAMETER - SET EQUAL TO WHAT ANTHONY HAS EXTRACTED.
double extractFixedTimeDuration = 4.0; // fixed length duration of all extracts from the original data - centred on the bounding box.
FileInfo sourceRecording = dataRecord.Path;
int minHz = dataRecord.LowFrequencyHertz;
int maxHz = dataRecord.HighFrequencyHertz;
TimeSpan eventDuration = dataRecord.EventDurationSeconds;
TimeSpan eventCentre = TimeSpan.FromTicks(eventDuration.Ticks / 2);
TimeSpan extractDuration = TimeSpan.FromSeconds(extractFixedTimeDuration);
TimeSpan extractHalfDuration = TimeSpan.FromSeconds(extractFixedTimeDuration / 2);
TimeSpan localEventStart = TimeSpan.Zero;
TimeSpan localEventEnd = extractDuration;
if (eventDuration != TimeSpan.Zero && eventDuration < extractDuration)
{
localEventStart = extractHalfDuration - eventCentre;
localEventEnd = extractHalfDuration + eventCentre;
}
var result = AnalyseOneRecording(sourceRecording, configDict, localEventStart, localEventEnd, minHz, maxHz, opDir);
// add additional info to identify this recording
result.AudioEventId = dataRecord.AudioEventId;
result.SiteName = dataRecord.SiteName;
result.CommonTags = dataRecord.CommonTags;
return(result);
}
public static CsvDataRecord ReadDataRecord(string record)
{
CsvDataRecord csvDataRecord = new CsvDataRecord();
// split and parse elements of data line
var fields = record.Split(',');
for (int i = 0; i < fields.Length; i++)
{
string word = fields[i];
while (word.StartsWith("\"") || word.StartsWith(" "))
{
word = word.Substring(1, word.Length - 1);
}
while (word.EndsWith("\"") || word.EndsWith(" "))
{
word = word.Substring(0, word.Length - 1);
}
fields[i] = word;
}
csvDataRecord.WavFileName = fields[0];
csvDataRecord.LowFrequencyHertz = (int)Math.Round(double.Parse(fields[1]));
csvDataRecord.HighFrequencyHertz = (int)Math.Round(double.Parse(fields[2]));
csvDataRecord.EventStartSeconds = TimeSpan.FromSeconds(double.Parse(fields[3]));
csvDataRecord.EventEndSeconds = TimeSpan.FromSeconds(double.Parse(fields[4]));
csvDataRecord.EventDurationSeconds = TimeSpan.FromSeconds(double.Parse(fields[5]));
return(csvDataRecord);
}
/// <summary>
/// Parses a value when it matches a specified pattern
/// </summary>
/// <param name="currentField">the current field that was read from a csv</param>
private void TryParseRaw(CsvDataRecord currentField)
{
try
{
if (!string.IsNullOrEmpty(currentField.RawText))
{
var conversion = importConfiguration.TypeConversions.FirstOrDefault(n => Regex.IsMatch(currentField.RawText, n.RawValuePattern, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline));
if (conversion != null)
{
currentField.Converted = ExpressionParser.Parse(conversion.ParseExpression, currentField, i => DefaultCallbacks.PrepareDefaultCallbacks(i.Scope, i.ReplSession));
}
}
}
catch (Exception ex)
{
LogEnvironment.LogEvent($"Unable to parse the value {currentField.RawText}. {ex.Message}", LogSeverity.Warning);
}
}
/// <summary>
/// Parses a single line
/// </summary>
/// <param name="s">the string that was read from the csv-file</param>
/// <param name="row">the current row</param>
private void ParseLine(string s, IDictionary <string, CsvDataRecord> row)
{
var matches = fieldRegex.Matches(s);
for (int i = 0; i < matches.Count; i++)
{
var name = $"col{i}";
CsvDataRecord currentField = new CsvDataRecord();
var rawValue = matches[i].Groups["value"].Value;
if (!string.IsNullOrEmpty(importConfiguration.EscapeStrategyName))
{
rawValue = EscapeStrategy.UnescapeString(importConfiguration.EscapeStrategyName, rawValue);
}
currentField.RawText = rawValue;
TryParseRaw(currentField);
row.Add(name, currentField);
}
}
//download_success { get; set; }
//skipped { get; set; }
public static CsvDataRecord ReadLine(string record)
{
CsvDataRecord csvDataRecord = new CsvDataRecord();
// split and parse elements of data line
var fields = record.Split(',');
for (int i = 0; i < fields.Length; i++)
{
string word = fields[i];
while (word.StartsWith("\"") || word.StartsWith(" "))
{
word = word.Substring(1, word.Length - 1);
}
while (word.EndsWith("\"") || word.EndsWith(" "))
{
word = word.Substring(0, word.Length - 1);
}
fields[i] = word;
}
csvDataRecord.AudioEventId = int.Parse(fields[0]);
csvDataRecord.AudioRecordingId = int.Parse(fields[1]);
csvDataRecord.Projects = fields[4];
csvDataRecord.SiteId = int.Parse(fields[5]);
csvDataRecord.SiteName = fields[6];
csvDataRecord.EventStartSeconds = TimeSpan.FromSeconds(double.Parse(fields[8]));
csvDataRecord.EventDurationSeconds = TimeSpan.FromSeconds(double.Parse(fields[10]));
csvDataRecord.PaddingStartTimeSeconds = TimeSpan.FromSeconds(double.Parse(fields[13]));
csvDataRecord.PaddingEndTimeSeconds = TimeSpan.FromSeconds(double.Parse(fields[14]));
csvDataRecord.LowFrequencyHertz = (int)Math.Round(double.Parse(fields[11]));
csvDataRecord.HighFrequencyHertz = (int)Math.Round(double.Parse(fields[12]));
csvDataRecord.CommonTags = fields[15];
csvDataRecord.SpeciesTags = fields[16];
csvDataRecord.Path = fields[20].ToFileInfo();
return(csvDataRecord);
}
/// <summary>
/// This method written 18-09-2014 to process Mangalam's CNN recordings.
/// Calculate the SNR statistics for each recording and then write info back to csv file
/// </summary>
public static void Main(Arguments arguments)
{
// set preprocessing parameter
bool doPreprocessing = true;
var output = arguments.Output;
if (!output.Exists)
{
output.Create();
}
Log.InfoFormat("# PRE-PROCESS SHORT AUDIO RECORDINGS FOR Convolutional DNN");
Log.InfoFormat("# DATE AND TIME: " + DateTime.Now);
Log.InfoFormat("# Input .csv file: " + arguments.Source);
Log.InfoFormat("# Configure file: " + arguments.Config);
Log.InfoFormat("# Output directry: " + arguments.Output);
// 1. set up the necessary files
FileInfo csvFileInfo = arguments.Source;
FileInfo configFile = arguments.Config.ToFileInfo();
// 2. get the config dictionary
var configDict = GetConfigurationForConvCnn(configFile);
// print out the parameters
LoggedConsole.WriteLine("\nPARAMETERS");
foreach (var kvp in configDict)
{
LoggedConsole.WriteLine("{0} = {1}", kvp.Key, kvp.Value);
}
// set up header of the output file
string outputPath = Path.Combine(output.FullName, "SNRInfoForConvDnnDataset.csv");
using (StreamWriter writer = new StreamWriter(outputPath))
{
string header = AudioToSonogramResult.GetCsvHeader();
writer.WriteLine(header);
}
// following int's are counters to monitor file availability
int lineNumber = 0;
int fileExistsCount = 0;
int fileLocationNotInCsv = 0;
int fileInCsvDoesNotExist = 0;
// keep track of species names and distribution of classes.
// following dictionaries are to monitor species numbers
var speciesCounts = new SpeciesCounts();
// read through the csv file containing info about recording locations and call bounds
try
{
var file = new FileStream(csvFileInfo.FullName, FileMode.Open);
var sr = new StreamReader(file);
// read the header and discard
string strLine;
lineNumber++;
while ((strLine = sr.ReadLine()) != null)
{
lineNumber++;
if (lineNumber % 5000 == 0)
{
Console.WriteLine(lineNumber);
}
// cannot use next line because reads the entire file
////var data = Csv.ReadFromCsv<string[]>(csvFileInfo).ToList();
// read single record from csv file
var record = CsvDataRecord.ReadLine(strLine);
if (record.Path == null)
{
fileLocationNotInCsv++;
////string warning = String.Format("######### WARNING: line {0} NULL PATH FIELD >>>null<<<", count);
////LoggedConsole.WriteWarnLine(warning);
continue;
}
var sourceRecording = record.Path;
var sourceDirectory = sourceRecording.Directory;
string parentDirectoryName = sourceDirectory.Parent.Name;
//var imageOpDir = new DirectoryInfo(output.FullName + @"\" + parentDirectoryName);
////DirectoryInfo imageOpDir = new DirectoryInfo(outDirectory.FullName + @"\" + parentDirectoryName + @"\" + directoryName);
////DirectoryInfo localSourceDir = new DirectoryInfo(@"C:\SensorNetworks\WavFiles\ConvDNNData");
////sourceRecording = Path.Combine(localSourceDir.FullName + @"\" + parentDirectoryName + @"\" + directoryName, fileName).ToFileInfo();
////record.path = sourceRecording;
/* ####################################### */
// TO TEST SMALL PORTION OF DATA because total data is too large.
doPreprocessing = false || parentDirectoryName.Equals("0");
if (!sourceRecording.Exists)
{
fileInCsvDoesNotExist++;
string warning = string.Format("FILE DOES NOT EXIST >>>," + sourceRecording.Name);
using (var writer = new StreamWriter(outputPath, true))
{
writer.WriteLine(warning);
}
continue;
}
// ####################################################################
if (doPreprocessing)
{
AudioToSonogramResult result = AnalyseOneRecording(record, configDict, output);
string line = result.WriteResultAsLineOfCsv();
// It is helpful to write to the output file as we go, so as to keep a record of where we are up to.
// This requires to open and close the output file at each iteration
using (var writer = new StreamWriter(outputPath, true))
{
writer.WriteLine(line);
}
}
// everything should be OK - have jumped through all the hoops.
fileExistsCount++;
// keep track of species names and distribution of classes.
speciesCounts.AddSpeciesCount(record.CommonTags);
speciesCounts.AddSpeciesId(record.CommonTags, record.SpeciesTags);
speciesCounts.AddSiteName(record.SiteName);
} // end while()
string classDistributionOpPath = Path.Combine(output.FullName, "ClassDistributionsForConvDnnDataset.csv");
speciesCounts.Save(classDistributionOpPath);
}
catch (IOException e)
{
LoggedConsole.WriteLine("Something went seriously bloody wrong!");
LoggedConsole.WriteLine(e.ToString());
return;
}
LoggedConsole.WriteLine("fileLocationNotInCsv =" + fileLocationNotInCsv);
LoggedConsole.WriteLine("fileInCsvDoesNotExist=" + fileInCsvDoesNotExist);
LoggedConsole.WriteLine("fileExistsCount =" + fileExistsCount);
LoggedConsole.WriteLine("\n##### FINISHED FILE ############################\n");
}
public static void Main(Arguments arguments)
{
var output = arguments.Output;
if (!output.Exists)
{
output.Create();
}
const string title = "# PRE-PROCESS SHORT AUDIO RECORDINGS FOR Convolutional DNN";
string date = "# DATE AND TIME: " + DateTime.Now;
LoggedConsole.WriteLine(title);
LoggedConsole.WriteLine(date);
LoggedConsole.WriteLine("# Input Query file: " + arguments.QueryWavFile);
LoggedConsole.WriteLine("# Input target file: " + arguments.TargtWavFile);
LoggedConsole.WriteLine("# Configure file: " + arguments.Config);
LoggedConsole.WriteLine("# Output directory: " + output.Name);
// 1. set up the necessary files
FileInfo queryWavfile = arguments.QueryWavFile.ToFileInfo();
FileInfo queryCsvfile = arguments.QueryCsvFile.ToFileInfo();
FileInfo targtWavfile = arguments.TargtWavFile.ToFileInfo();
FileInfo targtCsvfile = arguments.TargtCsvFile.ToFileInfo();
FileInfo configFile = arguments.Config.ToFileInfo();
DirectoryInfo opDir = output;
// 2. get the config dictionary
Config configuration = ConfigFile.Deserialize(configFile);
// below four lines are examples of retrieving info from Config config
//Config configuration = Yaml.Deserialise(configFile);
// string analysisIdentifier = configuration[AnalysisKeys.AnalysisName];
// int resampleRate = (int?)configuration[AnalysisKeys.ResampleRate] ?? AppConfigHelper.DefaultTargetSampleRate;
var configDict = new Dictionary <string, string>(configuration.ToDictionary());
configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
//bool makeSoxSonogram = (bool?)configuration[AnalysisKeys.MakeSoxSonogram] ?? false;
configDict[AnalysisKeys.AddTimeScale] = configuration[AnalysisKeys.AddTimeScale] ?? "true";
configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
// print out the parameters
LoggedConsole.WriteLine("\nPARAMETERS");
foreach (KeyValuePair <string, string> kvp in configDict)
{
LoggedConsole.WriteLine("{0} = {1}", kvp.Key, kvp.Value);
}
//set up the output file
//string header = "File Name, MinFreq(Hz), MaxFreq(Hz), StartTime(s), EndTime(s), Duration(s), Annotated by expert(Y-1/N-0),Correct Annotation(Y-1/N-0)";
string header = "File Name,MinFreq(Hz),MaxFreq(Hz),StartTime(s),EndTime(s),Duration(s),Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path2Spectrograms";
string opPath = Path.Combine(opDir.FullName, "OUTPUT.csv");
using (StreamWriter writer = new StreamWriter(opPath))
{
writer.WriteLine(header);
}
// reads the entire file
var data = FileTools.ReadTextFile(queryCsvfile.FullName);
// read single record from csv file
var record = CsvDataRecord.ReadDataRecord(data[1]);
if (!queryWavfile.Exists)
{
string warning = string.Format("FILE DOES NOT EXIST >>>," + arguments.QueryWavFile);
LoggedConsole.WriteWarnLine(warning);
return;
}
// ####################################################################
var result = AnalyseOneRecording(queryWavfile, configDict, record.EventStartSeconds, record.EventEndSeconds,
record.LowFrequencyHertz, record.HighFrequencyHertz, opDir);
// CONSTRUCT the outputline for csv file
// fileName,Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path
string line = string.Format("{0},{1},{2},{3:f2},{4:f2},{5:f2},{6:f1},{7:f3},{8:f3},{9:f3},{10}",
record.WavFileName, record.LowFrequencyHertz, record.HighFrequencyHertz,
record.EventStartSeconds.TotalSeconds, record.EventEndSeconds.TotalSeconds,
result.SnrStatistics.ExtractDuration.TotalSeconds,
result.SnrStatistics.Threshold, result.SnrStatistics.Snr,
result.SnrStatistics.FractionOfFramesExceedingThreshold, result.SnrStatistics.FractionOfFramesExceedingOneThirdSnr,
result.SpectrogramFile.FullName);
// It is helpful to write to the output file as we go, so as to keep a record of where we are up to.
// This requires to open and close the output file at each iteration
using (StreamWriter writer = new StreamWriter(opPath, true))
{
writer.WriteLine(line);
}
// ####################################################################
result = AnalyseOneRecording(targtWavfile, configDict, record.EventStartSeconds, record.EventEndSeconds,
record.LowFrequencyHertz, record.HighFrequencyHertz, opDir);
// CONSTRUCT the outputline for csv file
// fileName,Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path
//string line = String.Format("{0},{1},{2},{3:f2},{4:f2},{5:f2},{6:f1},{7:f3},{8:f3},{9:f3},{10}",
// record.wavFile_name, record.low_frequency_hertz, record.high_frequency_hertz,
// record.event_start_seconds.TotalSeconds, record.event_end_seconds.TotalSeconds,
// result.SnrStatistics.ExtractDuration.TotalSeconds,
// result.SnrStatistics.Threshold, result.SnrStatistics.Snr,
// result.SnrStatistics.FractionOfFramesExceedingThreshold, result.SnrStatistics.FractionOfFramesExceedingOneThirdSNR,
// result.SpectrogramFile.FullName);
// It is helpful to write to the output file as we go, so as to keep a record of where we are up to.
// This requires to open and close the output file at each iteration
using (StreamWriter writer = new StreamWriter(opPath, true))
{
writer.WriteLine(line);
}
} // end MAIN()
