// end MAIN() public static AudioToSonogramResult AnalyseOneRecording(CsvDataRecord dataRecord, Dictionary <string, string> configDict, DirectoryInfo opDir) { // ############## IMPORTANT PARAMETER - SET EQUAL TO WHAT ANTHONY HAS EXTRACTED. double extractFixedTimeDuration = 4.0; // fixed length duration of all extracts from the original data - centred on the bounding box. FileInfo sourceRecording = dataRecord.Path; int minHz = dataRecord.LowFrequencyHertz; int maxHz = dataRecord.HighFrequencyHertz; TimeSpan eventDuration = dataRecord.EventDurationSeconds; TimeSpan eventCentre = TimeSpan.FromTicks(eventDuration.Ticks / 2); TimeSpan extractDuration = TimeSpan.FromSeconds(extractFixedTimeDuration); TimeSpan extractHalfDuration = TimeSpan.FromSeconds(extractFixedTimeDuration / 2); TimeSpan localEventStart = TimeSpan.Zero; TimeSpan localEventEnd = extractDuration; if (eventDuration != TimeSpan.Zero && eventDuration < extractDuration) { localEventStart = extractHalfDuration - eventCentre; localEventEnd = extractHalfDuration + eventCentre; } var result = AnalyseOneRecording(sourceRecording, configDict, localEventStart, localEventEnd, minHz, maxHz, opDir); // add additional info to identify this recording result.AudioEventId = dataRecord.AudioEventId; result.SiteName = dataRecord.SiteName; result.CommonTags = dataRecord.CommonTags; return(result); }
public static CsvDataRecord ReadDataRecord(string record) { CsvDataRecord csvDataRecord = new CsvDataRecord(); // split and parse elements of data line var fields = record.Split(','); for (int i = 0; i < fields.Length; i++) { string word = fields[i]; while (word.StartsWith("\"") || word.StartsWith(" ")) { word = word.Substring(1, word.Length - 1); } while (word.EndsWith("\"") || word.EndsWith(" ")) { word = word.Substring(0, word.Length - 1); } fields[i] = word; } csvDataRecord.WavFileName = fields[0]; csvDataRecord.LowFrequencyHertz = (int)Math.Round(double.Parse(fields[1])); csvDataRecord.HighFrequencyHertz = (int)Math.Round(double.Parse(fields[2])); csvDataRecord.EventStartSeconds = TimeSpan.FromSeconds(double.Parse(fields[3])); csvDataRecord.EventEndSeconds = TimeSpan.FromSeconds(double.Parse(fields[4])); csvDataRecord.EventDurationSeconds = TimeSpan.FromSeconds(double.Parse(fields[5])); return(csvDataRecord); }
/// <summary> /// Parses a value when it matches a specified pattern /// </summary> /// <param name="currentField">the current field that was read from a csv</param> private void TryParseRaw(CsvDataRecord currentField) { try { if (!string.IsNullOrEmpty(currentField.RawText)) { var conversion = importConfiguration.TypeConversions.FirstOrDefault(n => Regex.IsMatch(currentField.RawText, n.RawValuePattern, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Multiline)); if (conversion != null) { currentField.Converted = ExpressionParser.Parse(conversion.ParseExpression, currentField, i => DefaultCallbacks.PrepareDefaultCallbacks(i.Scope, i.ReplSession)); } } } catch (Exception ex) { LogEnvironment.LogEvent($"Unable to parse the value {currentField.RawText}. {ex.Message}", LogSeverity.Warning); } }
/// <summary> /// Parses a single line /// </summary> /// <param name="s">the string that was read from the csv-file</param> /// <param name="row">the current row</param> private void ParseLine(string s, IDictionary <string, CsvDataRecord> row) { var matches = fieldRegex.Matches(s); for (int i = 0; i < matches.Count; i++) { var name = $"col{i}"; CsvDataRecord currentField = new CsvDataRecord(); var rawValue = matches[i].Groups["value"].Value; if (!string.IsNullOrEmpty(importConfiguration.EscapeStrategyName)) { rawValue = EscapeStrategy.UnescapeString(importConfiguration.EscapeStrategyName, rawValue); } currentField.RawText = rawValue; TryParseRaw(currentField); row.Add(name, currentField); } }
//download_success { get; set; } //skipped { get; set; } public static CsvDataRecord ReadLine(string record) { CsvDataRecord csvDataRecord = new CsvDataRecord(); // split and parse elements of data line var fields = record.Split(','); for (int i = 0; i < fields.Length; i++) { string word = fields[i]; while (word.StartsWith("\"") || word.StartsWith(" ")) { word = word.Substring(1, word.Length - 1); } while (word.EndsWith("\"") || word.EndsWith(" ")) { word = word.Substring(0, word.Length - 1); } fields[i] = word; } csvDataRecord.AudioEventId = int.Parse(fields[0]); csvDataRecord.AudioRecordingId = int.Parse(fields[1]); csvDataRecord.Projects = fields[4]; csvDataRecord.SiteId = int.Parse(fields[5]); csvDataRecord.SiteName = fields[6]; csvDataRecord.EventStartSeconds = TimeSpan.FromSeconds(double.Parse(fields[8])); csvDataRecord.EventDurationSeconds = TimeSpan.FromSeconds(double.Parse(fields[10])); csvDataRecord.PaddingStartTimeSeconds = TimeSpan.FromSeconds(double.Parse(fields[13])); csvDataRecord.PaddingEndTimeSeconds = TimeSpan.FromSeconds(double.Parse(fields[14])); csvDataRecord.LowFrequencyHertz = (int)Math.Round(double.Parse(fields[11])); csvDataRecord.HighFrequencyHertz = (int)Math.Round(double.Parse(fields[12])); csvDataRecord.CommonTags = fields[15]; csvDataRecord.SpeciesTags = fields[16]; csvDataRecord.Path = fields[20].ToFileInfo(); return(csvDataRecord); }
/// <summary> /// This method written 18-09-2014 to process Mangalam's CNN recordings. /// Calculate the SNR statistics for each recording and then write info back to csv file /// </summary> public static void Main(Arguments arguments) { // set preprocessing parameter bool doPreprocessing = true; var output = arguments.Output; if (!output.Exists) { output.Create(); } Log.InfoFormat("# PRE-PROCESS SHORT AUDIO RECORDINGS FOR Convolutional DNN"); Log.InfoFormat("# DATE AND TIME: " + DateTime.Now); Log.InfoFormat("# Input .csv file: " + arguments.Source); Log.InfoFormat("# Configure file: " + arguments.Config); Log.InfoFormat("# Output directry: " + arguments.Output); // 1. set up the necessary files FileInfo csvFileInfo = arguments.Source; FileInfo configFile = arguments.Config.ToFileInfo(); // 2. get the config dictionary var configDict = GetConfigurationForConvCnn(configFile); // print out the parameters LoggedConsole.WriteLine("\nPARAMETERS"); foreach (var kvp in configDict) { LoggedConsole.WriteLine("{0} = {1}", kvp.Key, kvp.Value); } // set up header of the output file string outputPath = Path.Combine(output.FullName, "SNRInfoForConvDnnDataset.csv"); using (StreamWriter writer = new StreamWriter(outputPath)) { string header = AudioToSonogramResult.GetCsvHeader(); writer.WriteLine(header); } // following int's are counters to monitor file availability int lineNumber = 0; int fileExistsCount = 0; int fileLocationNotInCsv = 0; int fileInCsvDoesNotExist = 0; // keep track of species names and distribution of classes. // following dictionaries are to monitor species numbers var speciesCounts = new SpeciesCounts(); // read through the csv file containing info about recording locations and call bounds try { var file = new FileStream(csvFileInfo.FullName, FileMode.Open); var sr = new StreamReader(file); // read the header and discard string strLine; lineNumber++; while ((strLine = sr.ReadLine()) != null) { lineNumber++; if (lineNumber % 5000 == 0) { Console.WriteLine(lineNumber); } // cannot use next line because reads the entire file ////var data = Csv.ReadFromCsv<string[]>(csvFileInfo).ToList(); // read single record from csv file var record = CsvDataRecord.ReadLine(strLine); if (record.Path == null) { fileLocationNotInCsv++; ////string warning = String.Format("######### WARNING: line {0} NULL PATH FIELD >>>null<<<", count); ////LoggedConsole.WriteWarnLine(warning); continue; } var sourceRecording = record.Path; var sourceDirectory = sourceRecording.Directory; string parentDirectoryName = sourceDirectory.Parent.Name; //var imageOpDir = new DirectoryInfo(output.FullName + @"\" + parentDirectoryName); ////DirectoryInfo imageOpDir = new DirectoryInfo(outDirectory.FullName + @"\" + parentDirectoryName + @"\" + directoryName); ////DirectoryInfo localSourceDir = new DirectoryInfo(@"C:\SensorNetworks\WavFiles\ConvDNNData"); ////sourceRecording = Path.Combine(localSourceDir.FullName + @"\" + parentDirectoryName + @"\" + directoryName, fileName).ToFileInfo(); ////record.path = sourceRecording; /* ####################################### */ // TO TEST SMALL PORTION OF DATA because total data is too large. doPreprocessing = false || parentDirectoryName.Equals("0"); if (!sourceRecording.Exists) { fileInCsvDoesNotExist++; string warning = string.Format("FILE DOES NOT EXIST >>>," + sourceRecording.Name); using (var writer = new StreamWriter(outputPath, true)) { writer.WriteLine(warning); } continue; } // #################################################################### if (doPreprocessing) { AudioToSonogramResult result = AnalyseOneRecording(record, configDict, output); string line = result.WriteResultAsLineOfCsv(); // It is helpful to write to the output file as we go, so as to keep a record of where we are up to. // This requires to open and close the output file at each iteration using (var writer = new StreamWriter(outputPath, true)) { writer.WriteLine(line); } } // everything should be OK - have jumped through all the hoops. fileExistsCount++; // keep track of species names and distribution of classes. speciesCounts.AddSpeciesCount(record.CommonTags); speciesCounts.AddSpeciesId(record.CommonTags, record.SpeciesTags); speciesCounts.AddSiteName(record.SiteName); } // end while() string classDistributionOpPath = Path.Combine(output.FullName, "ClassDistributionsForConvDnnDataset.csv"); speciesCounts.Save(classDistributionOpPath); } catch (IOException e) { LoggedConsole.WriteLine("Something went seriously bloody wrong!"); LoggedConsole.WriteLine(e.ToString()); return; } LoggedConsole.WriteLine("fileLocationNotInCsv =" + fileLocationNotInCsv); LoggedConsole.WriteLine("fileInCsvDoesNotExist=" + fileInCsvDoesNotExist); LoggedConsole.WriteLine("fileExistsCount =" + fileExistsCount); LoggedConsole.WriteLine("\n##### FINISHED FILE ############################\n"); }
public static void Main(Arguments arguments) { var output = arguments.Output; if (!output.Exists) { output.Create(); } const string title = "# PRE-PROCESS SHORT AUDIO RECORDINGS FOR Convolutional DNN"; string date = "# DATE AND TIME: " + DateTime.Now; LoggedConsole.WriteLine(title); LoggedConsole.WriteLine(date); LoggedConsole.WriteLine("# Input Query file: " + arguments.QueryWavFile); LoggedConsole.WriteLine("# Input target file: " + arguments.TargtWavFile); LoggedConsole.WriteLine("# Configure file: " + arguments.Config); LoggedConsole.WriteLine("# Output directory: " + output.Name); // 1. set up the necessary files FileInfo queryWavfile = arguments.QueryWavFile.ToFileInfo(); FileInfo queryCsvfile = arguments.QueryCsvFile.ToFileInfo(); FileInfo targtWavfile = arguments.TargtWavFile.ToFileInfo(); FileInfo targtCsvfile = arguments.TargtCsvFile.ToFileInfo(); FileInfo configFile = arguments.Config.ToFileInfo(); DirectoryInfo opDir = output; // 2. get the config dictionary Config configuration = ConfigFile.Deserialize(configFile); // below four lines are examples of retrieving info from Config config //Config configuration = Yaml.Deserialise(configFile); // string analysisIdentifier = configuration[AnalysisKeys.AnalysisName]; // int resampleRate = (int?)configuration[AnalysisKeys.ResampleRate] ?? AppConfigHelper.DefaultTargetSampleRate; var configDict = new Dictionary <string, string>(configuration.ToDictionary()); configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true"; configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true"; //bool makeSoxSonogram = (bool?)configuration[AnalysisKeys.MakeSoxSonogram] ?? false; configDict[AnalysisKeys.AddTimeScale] = configuration[AnalysisKeys.AddTimeScale] ?? "true"; configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true"; configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true"; // print out the parameters LoggedConsole.WriteLine("\nPARAMETERS"); foreach (KeyValuePair <string, string> kvp in configDict) { LoggedConsole.WriteLine("{0} = {1}", kvp.Key, kvp.Value); } //set up the output file //string header = "File Name, MinFreq(Hz), MaxFreq(Hz), StartTime(s), EndTime(s), Duration(s), Annotated by expert(Y-1/N-0),Correct Annotation(Y-1/N-0)"; string header = "File Name,MinFreq(Hz),MaxFreq(Hz),StartTime(s),EndTime(s),Duration(s),Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path2Spectrograms"; string opPath = Path.Combine(opDir.FullName, "OUTPUT.csv"); using (StreamWriter writer = new StreamWriter(opPath)) { writer.WriteLine(header); } // reads the entire file var data = FileTools.ReadTextFile(queryCsvfile.FullName); // read single record from csv file var record = CsvDataRecord.ReadDataRecord(data[1]); if (!queryWavfile.Exists) { string warning = string.Format("FILE DOES NOT EXIST >>>," + arguments.QueryWavFile); LoggedConsole.WriteWarnLine(warning); return; } // #################################################################### var result = AnalyseOneRecording(queryWavfile, configDict, record.EventStartSeconds, record.EventEndSeconds, record.LowFrequencyHertz, record.HighFrequencyHertz, opDir); // CONSTRUCT the outputline for csv file // fileName,Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path string line = string.Format("{0},{1},{2},{3:f2},{4:f2},{5:f2},{6:f1},{7:f3},{8:f3},{9:f3},{10}", record.WavFileName, record.LowFrequencyHertz, record.HighFrequencyHertz, record.EventStartSeconds.TotalSeconds, record.EventEndSeconds.TotalSeconds, result.SnrStatistics.ExtractDuration.TotalSeconds, result.SnrStatistics.Threshold, result.SnrStatistics.Snr, result.SnrStatistics.FractionOfFramesExceedingThreshold, result.SnrStatistics.FractionOfFramesExceedingOneThirdSnr, result.SpectrogramFile.FullName); // It is helpful to write to the output file as we go, so as to keep a record of where we are up to. // This requires to open and close the output file at each iteration using (StreamWriter writer = new StreamWriter(opPath, true)) { writer.WriteLine(line); } // #################################################################### result = AnalyseOneRecording(targtWavfile, configDict, record.EventStartSeconds, record.EventEndSeconds, record.LowFrequencyHertz, record.HighFrequencyHertz, opDir); // CONSTRUCT the outputline for csv file // fileName,Threshold,Snr,FractionOfFramesGTThreshold,FractionOfFramesGTThirdSNR,path //string line = String.Format("{0},{1},{2},{3:f2},{4:f2},{5:f2},{6:f1},{7:f3},{8:f3},{9:f3},{10}", // record.wavFile_name, record.low_frequency_hertz, record.high_frequency_hertz, // record.event_start_seconds.TotalSeconds, record.event_end_seconds.TotalSeconds, // result.SnrStatistics.ExtractDuration.TotalSeconds, // result.SnrStatistics.Threshold, result.SnrStatistics.Snr, // result.SnrStatistics.FractionOfFramesExceedingThreshold, result.SnrStatistics.FractionOfFramesExceedingOneThirdSNR, // result.SpectrogramFile.FullName); // It is helpful to write to the output file as we go, so as to keep a record of where we are up to. // This requires to open and close the output file at each iteration using (StreamWriter writer = new StreamWriter(opPath, true)) { writer.WriteLine(line); } } // end MAIN()