/// <summary> /// This method simply prints the overall metrics using EvaluateUtils.PrintConfusionMatrixAndPerFoldResults. /// Override if something else is needed. /// </summary> protected virtual void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics) { ch.AssertValue(ch); ch.AssertValue(metrics); IDataView fold; if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out fold)) { throw ch.Except("No overall metrics found"); } string weightedMetrics; string unweightedMetrics = MetricWriter.GetPerFoldResults(Host, fold, out weightedMetrics); if (!string.IsNullOrEmpty(weightedMetrics)) { ch.Info(weightedMetrics); } ch.Info(unweightedMetrics); }
protected override void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics) { ch.AssertValue(metrics); IDataView fold; if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out fold)) { throw ch.Except("No overall metrics found"); } // Show only the metrics for the requested index. fold = ExtractRelevantIndex(fold); string weightedMetrics; string unweightedMetrics = MetricWriter.GetPerFoldResults(Host, fold, out weightedMetrics); if (!string.IsNullOrEmpty(weightedMetrics)) { ch.Info(weightedMetrics); } ch.Info(unweightedMetrics); }
protected override void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics) { IDataView top; if (!metrics.TryGetValue(AnomalyDetectionEvaluator.TopKResults, out top)) { throw Host.Except("Did not find the top-k results data view"); } var sb = new StringBuilder(); using (var cursor = top.GetRowCursor(col => true)) { int index; if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.Instance, out index)) { throw Host.Except("Data view does not contain the 'Instance' column"); } var instanceGetter = cursor.GetGetter <ReadOnlyMemory <char> >(index); if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.AnomalyScore, out index)) { throw Host.Except("Data view does not contain the 'Anomaly Score' column"); } var scoreGetter = cursor.GetGetter <Single>(index); if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.Label, out index)) { throw Host.Except("Data view does not contain the 'Label' column"); } var labelGetter = cursor.GetGetter <Single>(index); bool hasRows = false; while (cursor.MoveNext()) { if (!hasRows) { sb.AppendFormat("{0} Top-scored Results", _topScored); sb.AppendLine(); sb.AppendLine("================================================="); sb.AppendLine("Instance Anomaly Score Labeled"); hasRows = true; } var name = default(ReadOnlyMemory <char>); Single score = 0; Single label = 0; instanceGetter(ref name); scoreGetter(ref score); labelGetter(ref label); sb.AppendFormat("{0,-10}{1,12:G4}{2,12}", name, score, label); sb.AppendLine(); } } if (sb.Length > 0) { ch.Info(MessageSensitivity.UserData, sb.ToString()); } IDataView overall; if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out overall)) { throw Host.Except("No overall metrics found"); } // Find the number of anomalies, and the thresholds. int numAnomIndex; if (!overall.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.OverallMetrics.NumAnomalies, out numAnomIndex)) { throw Host.Except("Could not find the 'NumAnomalies' column"); } int stratCol; var hasStrat = overall.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratCol, out stratCol); int stratVal; bool hasStratVals = overall.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratVal, out stratVal); Contracts.Assert(hasStrat == hasStratVals); long numAnomalies = 0; using (var cursor = overall.GetRowCursor(col => col == numAnomIndex || (hasStrat && col == stratCol))) { var numAnomGetter = cursor.GetGetter <long>(numAnomIndex); ValueGetter <uint> stratGetter = null; if (hasStrat) { var type = cursor.Schema.GetColumnType(stratCol); stratGetter = RowCursorUtils.GetGetterAs <uint>(type, cursor, stratCol); } bool foundRow = false; while (cursor.MoveNext()) { uint strat = 0; if (stratGetter != null) { stratGetter(ref strat); } if (strat > 0) { continue; } if (foundRow) { throw Host.Except("Found multiple non-stratified rows in overall results data view"); } foundRow = true; numAnomGetter(ref numAnomalies); } } var args = new ChooseColumnsTransform.Arguments(); var cols = new List <ChooseColumnsTransform.Column>() { new ChooseColumnsTransform.Column() { Name = string.Format(FoldDrAtKFormat, _k), Source = AnomalyDetectionEvaluator.OverallMetrics.DrAtK }, new ChooseColumnsTransform.Column() { Name = string.Format(FoldDrAtPFormat, _p), Source = AnomalyDetectionEvaluator.OverallMetrics.DrAtPFpr }, new ChooseColumnsTransform.Column() { Name = string.Format(FoldDrAtNumAnomaliesFormat, numAnomalies), Source = AnomalyDetectionEvaluator.OverallMetrics.DrAtNumPos }, new ChooseColumnsTransform.Column() { Name = AnomalyDetectionEvaluator.OverallMetrics.ThreshAtK }, new ChooseColumnsTransform.Column() { Name = AnomalyDetectionEvaluator.OverallMetrics.ThreshAtP }, new ChooseColumnsTransform.Column() { Name = AnomalyDetectionEvaluator.OverallMetrics.ThreshAtNumPos }, new ChooseColumnsTransform.Column() { Name = BinaryClassifierEvaluator.Auc } }; args.Column = cols.ToArray(); IDataView fold = new ChooseColumnsTransform(Host, args, overall); string weightedFold; ch.Info(MetricWriter.GetPerFoldResults(Host, fold, out weightedFold)); }