/// <summary>
/// This method simply prints the overall metrics using EvaluateUtils.PrintConfusionMatrixAndPerFoldResults.
/// Override if something else is needed.
/// </summary>
protected virtual void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics)
{
ch.AssertValue(ch);
ch.AssertValue(metrics);
IDataView fold;
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out fold))
{
throw ch.Except("No overall metrics found");
}
string weightedMetrics;
string unweightedMetrics = MetricWriter.GetPerFoldResults(Host, fold, out weightedMetrics);
if (!string.IsNullOrEmpty(weightedMetrics))
{
ch.Info(weightedMetrics);
}
ch.Info(unweightedMetrics);
}
protected override void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics)
{
ch.AssertValue(metrics);
IDataView fold;
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out fold))
{
throw ch.Except("No overall metrics found");
}
// Show only the metrics for the requested index.
fold = ExtractRelevantIndex(fold);
string weightedMetrics;
string unweightedMetrics = MetricWriter.GetPerFoldResults(Host, fold, out weightedMetrics);
if (!string.IsNullOrEmpty(weightedMetrics))
{
ch.Info(weightedMetrics);
}
ch.Info(unweightedMetrics);
}
protected override void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics)
{
IDataView top;
if (!metrics.TryGetValue(AnomalyDetectionEvaluator.TopKResults, out top))
{
throw Host.Except("Did not find the top-k results data view");
}
var sb = new StringBuilder();
using (var cursor = top.GetRowCursor(col => true))
{
int index;
if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.Instance, out index))
{
throw Host.Except("Data view does not contain the 'Instance' column");
}
var instanceGetter = cursor.GetGetter <ReadOnlyMemory <char> >(index);
if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.AnomalyScore, out index))
{
throw Host.Except("Data view does not contain the 'Anomaly Score' column");
}
var scoreGetter = cursor.GetGetter <Single>(index);
if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.Label, out index))
{
throw Host.Except("Data view does not contain the 'Label' column");
}
var labelGetter = cursor.GetGetter <Single>(index);
bool hasRows = false;
while (cursor.MoveNext())
{
if (!hasRows)
{
sb.AppendFormat("{0} Top-scored Results", _topScored);
sb.AppendLine();
sb.AppendLine("=================================================");
sb.AppendLine("Instance Anomaly Score Labeled");
hasRows = true;
}
var name = default(ReadOnlyMemory <char>);
Single score = 0;
Single label = 0;
instanceGetter(ref name);
scoreGetter(ref score);
labelGetter(ref label);
sb.AppendFormat("{0,-10}{1,12:G4}{2,12}", name, score, label);
sb.AppendLine();
}
}
if (sb.Length > 0)
{
ch.Info(MessageSensitivity.UserData, sb.ToString());
}
IDataView overall;
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out overall))
{
throw Host.Except("No overall metrics found");
}
// Find the number of anomalies, and the thresholds.
int numAnomIndex;
if (!overall.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.OverallMetrics.NumAnomalies, out numAnomIndex))
{
throw Host.Except("Could not find the 'NumAnomalies' column");
}
int stratCol;
var hasStrat = overall.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratCol, out stratCol);
int stratVal;
bool hasStratVals = overall.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratVal, out stratVal);
Contracts.Assert(hasStrat == hasStratVals);
long numAnomalies = 0;
using (var cursor = overall.GetRowCursor(col => col == numAnomIndex ||
(hasStrat && col == stratCol)))
{
var numAnomGetter = cursor.GetGetter <long>(numAnomIndex);
ValueGetter <uint> stratGetter = null;
if (hasStrat)
{
var type = cursor.Schema.GetColumnType(stratCol);
stratGetter = RowCursorUtils.GetGetterAs <uint>(type, cursor, stratCol);
}
bool foundRow = false;
while (cursor.MoveNext())
{
uint strat = 0;
if (stratGetter != null)
{
stratGetter(ref strat);
}
if (strat > 0)
{
continue;
}
if (foundRow)
{
throw Host.Except("Found multiple non-stratified rows in overall results data view");
}
foundRow = true;
numAnomGetter(ref numAnomalies);
}
}
var args = new ChooseColumnsTransform.Arguments();
var cols = new List <ChooseColumnsTransform.Column>()
{
new ChooseColumnsTransform.Column()
{
Name = string.Format(FoldDrAtKFormat, _k),
Source = AnomalyDetectionEvaluator.OverallMetrics.DrAtK
},
new ChooseColumnsTransform.Column()
{
Name = string.Format(FoldDrAtPFormat, _p),
Source = AnomalyDetectionEvaluator.OverallMetrics.DrAtPFpr
},
new ChooseColumnsTransform.Column()
{
Name = string.Format(FoldDrAtNumAnomaliesFormat, numAnomalies),
Source = AnomalyDetectionEvaluator.OverallMetrics.DrAtNumPos
},
new ChooseColumnsTransform.Column()
{
Name = AnomalyDetectionEvaluator.OverallMetrics.ThreshAtK
},
new ChooseColumnsTransform.Column()
{
Name = AnomalyDetectionEvaluator.OverallMetrics.ThreshAtP
},
new ChooseColumnsTransform.Column()
{
Name = AnomalyDetectionEvaluator.OverallMetrics.ThreshAtNumPos
},
new ChooseColumnsTransform.Column()
{
Name = BinaryClassifierEvaluator.Auc
}
};
args.Column = cols.ToArray();
IDataView fold = new ChooseColumnsTransform(Host, args, overall);
string weightedFold;
ch.Info(MetricWriter.GetPerFoldResults(Host, fold, out weightedFold));
}
