public static DataTable AnalysisReturnsDataTable(int iter, FileInfo fiSegmentOfSourceFile, Dictionary <string, string> configDict, DirectoryInfo diOutputDir)
{
string opFileName = "temp.wav";
//######################################################################
var results = Analysis(fiSegmentOfSourceFile, configDict, diOutputDir, opFileName, TimeSpan.Zero);
//######################################################################
if (results == null)
{
return(null);
}
var sonogram = results.Item1;
var hits = results.Item2;
var scores = results.Item3;
var predictedEvents = results.Item4;
var recordingTimeSpan = results.Item5;
double segmentDuration = double.Parse(configDict[key_SEGMENT_DURATION]);
double segmentStartMinute = segmentDuration * iter;
DataTable dataTable = null;
if ((predictedEvents == null) || (predictedEvents.Count == 0))
{
LoggedConsole.WriteLine("############ WARNING: No acoustic events were returned from the analysis.");
}
else
{
string analysisName = configDict[key_ANALYSIS_NAME];
string fName = Path.GetFileNameWithoutExtension(fiSegmentOfSourceFile.Name);
foreach (AcousticEvent ev in predictedEvents)
{
ev.FileName = fName;
//ev.Name = analysisName; //name is the periodicity
ev.SegmentDurationSeconds = recordingTimeSpan.TotalSeconds;
}
//write events to a data table to return.
dataTable = WriteEvents2DataTable(segmentStartMinute, recordingTimeSpan, predictedEvents);
string sortString = key_START_ABS + " ASC";
dataTable = DataTableTools.SortTable(dataTable, sortString); //sort by start time before returning
}
//draw images of sonograms
int DRAW_SONOGRAMS = int.Parse(configDict[key_DRAW_SONOGRAMS]); // options to draw sonogram
bool saveSonogram = false;
if ((DRAW_SONOGRAMS == 2) || ((DRAW_SONOGRAMS == 1) && (predictedEvents.Count > 0)))
{
saveSonogram = true;
}
if (saveSonogram)
{
double eventThreshold = 0.1;
string imagePath = Path.Combine(diOutputDir.FullName, Path.GetFileNameWithoutExtension(fiSegmentOfSourceFile.FullName) + "_" + (int)segmentStartMinute + "min.png");
Image image = DrawSonogram(sonogram, hits, scores, predictedEvents, eventThreshold);
image.Save(imagePath, ImageFormat.Png);
}
return(dataTable);
}
public static DataTable MergeAdjacentPredictions(DataTable dt)
{
//DataTable newTable = DataTableTools.CreateTable(dt);
string sortString = AnalysisKeys.EventStartAbs + " ASC";
dt = DataTableTools.SortTable(dt, sortString);
int rowCount = dt.Rows.Count;
// work from end to beginning
for (int i = rowCount - 2; i >= 0; i--)
{
DataRow row1 = dt.Rows[i];
DataRow row2 = dt.Rows[i + 1];
string name1 = (string)row1[AnalysisKeys.EventName];
string name2 = (string)row2[AnalysisKeys.EventName];
string predictedSex1;
if (name1.EndsWith("(m)"))
{
predictedSex1 = "M";
}
else if (name1.EndsWith("(f)"))
{
predictedSex1 = "F";
}
else
{
predictedSex1 = null;
}
string predictedSex2;
if (name2.EndsWith("(m)"))
{
predictedSex2 = "M";
}
else if (name2.EndsWith("(f)"))
{
predictedSex2 = "F";
}
else
{
predictedSex2 = null;
}
double start1 = (double)row1[AnalysisKeys.EventStartAbs];
double start2 = (double)row2[AnalysisKeys.EventStartAbs];
if (start2 - start1 < 15.0 && predictedSex1 == predictedSex2)
{
dt.Rows.Remove(row2);
}
}
return(dt);
}
} //AddContext2Table()
public Tuple <DataTable, DataTable> ProcessCsvFile(FileInfo fiCsvFile, FileInfo fiConfigFile)
{
DataTable dt = CsvTools.ReadCSVToTable(fiCsvFile.FullName, true); //get original data table
if ((dt == null) || (dt.Rows.Count == 0))
{
return(null);
}
//get its column headers
var dtHeaders = new List <string>();
var dtTypes = new List <Type>();
foreach (DataColumn col in dt.Columns)
{
dtHeaders.Add(col.ColumnName);
dtTypes.Add(col.DataType);
}
List <string> displayHeaders = null;
//check if config file contains list of display headers
if (fiConfigFile != null)
{
var configuration = new ConfigDictionary(fiConfigFile.FullName);
Dictionary <string, string> configDict = configuration.GetTable();
if (configDict.ContainsKey(Keys.DISPLAY_COLUMNS))
{
displayHeaders = configDict[Keys.DISPLAY_COLUMNS].Split(',').ToList();
}
}
//if config file does not exist or does not contain display headers then use the original headers
if (displayHeaders == null)
{
displayHeaders = dtHeaders; //use existing headers if user supplies none.
}
//now determine how to display tracks in display datatable
Type[] displayTypes = new Type[displayHeaders.Count];
bool[] canDisplay = new bool[displayHeaders.Count];
for (int i = 0; i < displayTypes.Length; i++)
{
displayTypes[i] = typeof(double);
canDisplay[i] = false;
if (dtHeaders.Contains(displayHeaders[i]))
{
canDisplay[i] = true;
}
}
DataTable table2Display = DataTableTools.CreateTable(displayHeaders.ToArray(), displayTypes);
foreach (DataRow row in dt.Rows)
{
DataRow newRow = table2Display.NewRow();
for (int i = 0; i < canDisplay.Length; i++)
{
if (canDisplay[i])
{
newRow[displayHeaders[i]] = row[displayHeaders[i]];
}
else
{
newRow[displayHeaders[i]] = 0.0;
}
}
table2Display.Rows.Add(newRow);
}
//order the table if possible
if (dt.Columns.Contains(AudioAnalysisTools.Keys.EVENT_START_ABS))
{
dt = DataTableTools.SortTable(dt, AudioAnalysisTools.Keys.EVENT_START_ABS + " ASC");
}
else if (dt.Columns.Contains(AudioAnalysisTools.Keys.EVENT_COUNT))
{
dt = DataTableTools.SortTable(dt, AudioAnalysisTools.Keys.EVENT_COUNT + " ASC");
}
else if (dt.Columns.Contains(AudioAnalysisTools.Keys.INDICES_COUNT))
{
dt = DataTableTools.SortTable(dt, AudioAnalysisTools.Keys.INDICES_COUNT + " ASC");
}
else if (dt.Columns.Contains(AudioAnalysisTools.Keys.START_MIN))
{
dt = DataTableTools.SortTable(dt, AudioAnalysisTools.Keys.START_MIN + " ASC");
}
table2Display = NormaliseColumnsOfDataTable(table2Display);
return(System.Tuple.Create(dt, table2Display));
} // ProcessCsvFile()
public AnalysisResult Analyse(AnalysisSettings analysisSettings)
{
//var configuration = new ConfigDictionary(analysisSettings.ConfigFile.FullName);
//Dictionary<string, string> configDict = configuration.GetTable();
var fiAudioF = analysisSettings.AudioFile;
var diOutputDir = analysisSettings.AnalysisRunDirectory;
var analysisResults = new AnalysisResult();
analysisResults.AnalysisIdentifier = this.Identifier;
analysisResults.SettingsUsed = analysisSettings;
analysisResults.Data = null;
//######################################################################
var results = Analysis(fiAudioF, analysisSettings.ConfigDict);
//######################################################################
if (results == null)
{
return(analysisResults); //nothing to process
}
var sonogram = results.Item1;
var hits = results.Item2;
var scores = results.Item3;
var predictedEvents = results.Item4;
var recordingTimeSpan = results.Item5;
analysisResults.AudioDuration = recordingTimeSpan;
DataTable dataTableOfEvents = null;
if ((predictedEvents != null) && (predictedEvents.Count != 0))
{
string analysisName = analysisSettings.ConfigDict[AudioAnalysisTools.Keys.ANALYSIS_NAME];
string fName = Path.GetFileNameWithoutExtension(fiAudioF.Name);
foreach (AcousticEvent ev in predictedEvents)
{
ev.SourceFileName = fName;
//ev.Name = analysisName; //name was added previously
ev.SourceFileDuration = recordingTimeSpan.TotalSeconds;
}
//write events to a data table to return.
dataTableOfEvents = WriteEvents2DataTable(predictedEvents);
string sortString = AudioAnalysisTools.Keys.EVENT_START_SEC + " ASC";
dataTableOfEvents = DataTableTools.SortTable(dataTableOfEvents, sortString); //sort by start time before returning
}
if ((analysisSettings.EventsFile != null) && (dataTableOfEvents != null))
{
CsvTools.DataTable2CSV(dataTableOfEvents, analysisSettings.EventsFile.FullName);
}
if ((analysisSettings.IndicesFile != null) && (dataTableOfEvents != null))
{
double scoreThreshold = 0.1;
TimeSpan unitTime = TimeSpan.FromSeconds(60); //index for each time span of i minute
var indicesDT = ConvertEvents2Indices(dataTableOfEvents, unitTime, recordingTimeSpan, scoreThreshold);
CsvTools.DataTable2CSV(indicesDT, analysisSettings.IndicesFile.FullName);
}
//save image of sonograms
if ((sonogram != null) && (analysisSettings.ImageFile != null))
{
var fileExists = File.Exists(analysisSettings.ImageFile.FullName);
string imagePath = analysisSettings.ImageFile.FullName;
double eventThreshold = 0.1;
Image image = DrawSonogram(sonogram, hits, scores, predictedEvents, eventThreshold);
//image.Save(imagePath, ImageFormat.Png);
lock (imageWriteLock)
{
//try
//{
image.Save(analysisSettings.ImageFile.FullName, ImageFormat.Png);
//}
//catch (Exception ex)
//{
//}
}
}
analysisResults.Data = dataTableOfEvents;
analysisResults.ImageFile = analysisSettings.ImageFile;
analysisResults.AudioDuration = recordingTimeSpan;
//result.DisplayItems = { { 0, "example" }, { 1, "example 2" }, }
//result.OutputFiles = { { "exmaple file key", new FileInfo("Where's that file?") } }
return(analysisResults);
} //Analyse()
public Tuple <DataTable, DataTable> ProcessCsvFile(FileInfo fiCsvFile, FileInfo fiConfigFile)
{
DataTable dt = CsvTools.ReadCSVToTable(fiCsvFile.FullName, true); //get original data table
if ((dt == null) || (dt.Rows.Count == 0))
{
return(null);
}
//get its column headers
var dtHeaders = new List <string>();
var dtTypes = new List <Type>();
foreach (DataColumn col in dt.Columns)
{
dtHeaders.Add(col.ColumnName);
dtTypes.Add(col.DataType);
}
List <string> displayHeaders = null;
//check if config file contains list of display headers
if (fiConfigFile != null)
{
var configuration = new ConfigDictionary(fiConfigFile.FullName);
Dictionary <string, string> configDict = configuration.GetTable();
if (configDict.ContainsKey(AnalysisKeys.DisplayColumns))
{
displayHeaders = configDict[AnalysisKeys.DisplayColumns].Split(',').ToList();
}
}
//if config file does not exist or does not contain display headers then use the original headers
if (displayHeaders == null)
{
displayHeaders = dtHeaders; //use existing headers if user supplies none.
}
//now determine how to display tracks in display datatable
Type[] displayTypes = new Type[displayHeaders.Count];
bool[] canDisplay = new bool[displayHeaders.Count];
for (int i = 0; i < displayTypes.Length; i++)
{
displayTypes[i] = typeof(double);
canDisplay[i] = false;
if (dtHeaders.Contains(displayHeaders[i]))
{
canDisplay[i] = true;
}
}
DataTable table2Display = DataTableTools.CreateTable(displayHeaders.ToArray(), displayTypes);
foreach (DataRow row in dt.Rows)
{
DataRow newRow = table2Display.NewRow();
for (int i = 0; i < canDisplay.Length; i++)
{
if (canDisplay[i])
{
newRow[displayHeaders[i]] = row[displayHeaders[i]];
}
else
{
newRow[displayHeaders[i]] = 0.0;
}
}
table2Display.Rows.Add(newRow);
}
//order the table if possible
if (dt.Columns.Contains(AnalysisKeys.EventStartAbs))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.EventStartAbs + " ASC");
}
else if (dt.Columns.Contains(AnalysisKeys.EventCount))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.EventCount + " ASC");
}
else if (dt.Columns.Contains(AnalysisKeys.KeyRankOrder))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.KeyRankOrder + " ASC");
}
else if (dt.Columns.Contains(AnalysisKeys.KeyStartMinute))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.KeyStartMinute + " ASC");
}
//this depracted now that use class indexProperties to do normalisation
//table2Display = NormaliseColumnsOfDataTable(table2Display);
//add in column of weighted indices
//bool addColumnOfweightedIndices = true;
//if (addColumnOfweightedIndices)
//{
// double[] comboWts = IndexCalculate.CalculateComboWeights();
// double[] weightedIndices = IndexCalculate.GetArrayOfWeightedAcousticIndices(dt, comboWts);
// string colName = "WeightedIndex";
// DataTableTools.AddColumnOfDoubles2Table(table2Display, colName, weightedIndices);
//}
return(Tuple.Create(dt, table2Display));
} // ProcessCsvFile()
/// <summary>
/// Calculates an ROC score for the predictions and tags provided in the passed data table.
/// First order the data by appropriate score as per the sort string
/// </summary>
/// <param name="dt"></param>
/// <param name="countOfTargetPositives"></param>
/// <param name="predictionCount"></param>
public static void ROCCurve(DataTable dt, int totalPositiveCount, int totalNegativeCount, string sortString)
{
dt = DataTableTools.SortTable(dt, sortString);
double previousRecall = 0.0;
int cumulativeTP = 0;
int cumulativeFP = 0;
double area = 0.0; //area under the ROC curve
List <double> ROC_Curve = new List <double>();
double maxAccuracy = 0.0;
double precisionAtMax = 0.0;
double specificityAtMax = 0.0;
double recallAtMax = 0.0;
double scoreAtMax = 0.0;
int optimumCount = 0;
//double precisionAt30 = 0.0;
//double recallAt30 = 0.0;
//double scoreAt30 = 0.0;
int count = 0;
foreach (DataRow row in dt.Rows)
{
int value = (int)row["TP"];
if (value == 1)
{
cumulativeTP++;
}
else
if ((int)row["FP"] == 1)
{
cumulativeFP++;
}
double recall = cumulativeTP / (double)totalPositiveCount; //the true positive rate
double specificity = cumulativeFP / (double)totalNegativeCount;
double precision = cumulativeTP / (double)(cumulativeTP + cumulativeFP);
double accuracy = (recall + precision) / 2;
if (accuracy > maxAccuracy)
{
optimumCount = count;
maxAccuracy = accuracy;
recallAtMax = recall;
precisionAtMax = precision;
specificityAtMax = specificity;
scoreAtMax = (double)row[AnalysisKeys.EventNormscore];
}
count++;
//if (count == 30)
//{
// recallAt30 = recall;
// precisionAt30 = precision;
// scoreAt30 = (double)row[Keys.EVENT_NORMSCORE];
//}
//double delta = precision * (recall - previousRecall);
double delta = specificity * (recall - previousRecall);
//double fpRate = 1 - specificity;
//double delta = fpRate * (recall - previousRecall);
area += delta;
if (delta > 0.0)
{
ROC_Curve.Add(delta);
}
previousRecall = recall;
} //foreach row in table
if (ROC_Curve.Count > 0)
{
DataTools.writeBarGraph(ROC_Curve.ToArray());
LoggedConsole.WriteLine("Area under ROC curve = {0:f4}", area);
LoggedConsole.WriteLine("Max accuracy={0:f3} for score threshold={1:f3}", maxAccuracy, scoreAtMax);
LoggedConsole.WriteLine(" where recall={0:f3}, precision={1:f3}, specifcity={2:f3}", recallAtMax, precisionAtMax, specificityAtMax);
//LoggedConsole.WriteLine("At 30 samples: recall={0:f3}, precision={1:f3}, at score={2:f3}", recallAt30, precisionAt30, scoreAt30);
}
}
public override AnalysisResult2 Analyze <T>(AnalysisSettings analysisSettings, SegmentSettings <T> segmentSettings)
{
var fiAudioF = segmentSettings.SegmentAudioFile;
var diOutputDir = segmentSettings.SegmentOutputDirectory;
//######################################################################
var results = Analysis(fiAudioF, analysisSettings, segmentSettings.Segment.SourceMetadata.SampleRate, segmentSettings.SegmentStartOffset);
//######################################################################
if (results == null)
{
return(null); //nothing to process (broken)
}
var sonogram = results.Item1;
var hits = results.Item2;
var scores = results.Item3;
var predictedEvents = results.Item4;
var recordingTimeSpan = results.Item5;
var result = new AnalysisResult2(analysisSettings, segmentSettings, recordingTimeSpan);
result.AnalysisIdentifier = this.Identifier;
result.MiscellaneousResults["dataTable"] = null;
DataTable dataTable = null;
if (predictedEvents != null)
{
string analysisName = analysisSettings.ConfigDict[AnalysisKeys.AnalysisName];
string fName = Path.GetFileNameWithoutExtension(fiAudioF.Name);
foreach (AcousticEvent ev in predictedEvents)
{
ev.FileName = fName;
//ev.Name = analysisName; //TEMPORARY DISABLE
ev.SegmentDurationSeconds = recordingTimeSpan.TotalSeconds;
}
//write events to a data table to return.
dataTable = WriteEvents2DataTable(predictedEvents);
string sortString = AnalysisKeys.EventStartAbs + " ASC";
dataTable = DataTableTools.SortTable(dataTable, sortString); //sort by start time before returning
}
if (analysisSettings.AnalysisDataSaveBehavior)
{
CsvTools.DataTable2CSV(dataTable, segmentSettings.SegmentEventsFile.FullName);
}
else
{
result.EventsFile = null;
}
if (analysisSettings.AnalysisDataSaveBehavior)
{
double scoreThreshold = 0.01;
if (analysisSettings.ConfigDict.ContainsKey(AnalysisKeys.IntensityThreshold))
{
scoreThreshold = ConfigDictionary.GetDouble(AnalysisKeys.IntensityThreshold, analysisSettings.ConfigDict);
}
TimeSpan unitTime = TimeSpan.FromSeconds(60); //index for each time span of i minute
var indicesDT = this.ConvertEvents2Indices(dataTable, unitTime, recordingTimeSpan, scoreThreshold);
CsvTools.DataTable2CSV(indicesDT, segmentSettings.SegmentSummaryIndicesFile.FullName);
}
else
{
result.SummaryIndices = null;
}
//save image of sonograms
if (analysisSettings.AnalysisImageSaveBehavior.ShouldSave(predictedEvents.Count))
{
string imagePath = segmentSettings.SegmentImageFile.FullName;
Image image = DrawSonogram(sonogram, hits, scores, predictedEvents);
image.Save(imagePath, ImageFormat.Png);
}
result.MiscellaneousResults["dataTable"] = dataTable;
result.ImageFile = segmentSettings.SegmentImageFile;
//result.DisplayItems = { { 0, "example" }, { 1, "example 2" }, }
//result.OutputFiles = { { "exmaple file key", new FileInfo("Where's that file?") } }
return(result);
}
