/// <summary>
/// Checks that NLU results do not regress performance against the baseline beyond the given thresholds.
/// </summary>
/// <param name="currentResults">Current results.</param>
/// <param name="baselineResults">Baseline results.</param>
/// <param name="threshold">Performance threshold.</param>
/// <returns>
/// <code>true</code> if performance has not dropped beyond thresholds, otherwise <code>false</code>.
/// </returns>
public static bool CheckThreshold(this NLUStatistics currentResults, NLUStatistics baselineResults, NLUThreshold threshold)
{
ConfusionMatrix getResultsByGroup(string group, IReadOnlyDictionary <string, ConfusionMatrix> groupStatistics)
{
if (groupStatistics != null && groupStatistics.TryGetValue(group, out var result))
{
return(result);
}
return(null);
}
ConfusionMatrix getResults(NLUStatistics statistics)
{
if (threshold.Type == "intent" || threshold.Type == "entity")
{
if (threshold.Group == null || threshold.Group == "*")
{
return(threshold.Type == "intent"
? statistics?.Intent
: statistics?.Entity);
}
return(threshold.Type == "intent"
? getResultsByGroup(threshold.Group, statistics?.ByIntent)
: getResultsByGroup(threshold.Group, statistics?.ByEntityType));
}
return(null);
}
var currentMatrix = getResults(currentResults) ?? ConfusionMatrix.Default;
if (threshold.Comparison == NLUThresholdKind.Absolute)
{
return(currentMatrix.GetMetric(threshold.Metric) >= threshold.Threshold);
}
var baselineMatrix = getResults(baselineResults) ?? ConfusionMatrix.Default;
return(baselineMatrix.GetMetric(threshold.Metric) - currentMatrix.GetMetric(threshold.Metric) <= threshold.Threshold);
}
/// <summary>
/// Prints to the console the intents, entities performance results
/// and a confusion table for intents
/// </summary>
/// <param name="compareResults"> The comparison results for intents and entities</param>
/// <param name="baseline">The baseline results the results are benchmarked against.</param>
public static void PrintResults(this NLUCompareResults compareResults, NLUStatistics baseline)
{
var allIntentsPrecision = Print(compareResults.Statistics.Intent.Precision(), baseline?.Intent.Precision());
var allIntentsRecall = Print(compareResults.Statistics.Intent.Recall(), baseline?.Intent.Recall());
var allIntentsF1 = Print(compareResults.Statistics.Intent.F1(), baseline?.Intent.F1());
var allIntentsTotal = Print(compareResults.Statistics.Intent.Total(), baseline?.Intent.Total(), 0);
var allIntentsFP = Print(compareResults.Statistics.Intent.FalsePositive, baseline?.Intent.FalsePositive, 0);
Console.WriteLine("## Intents Results");
var intentTable = new ConsoleTable("Intent", "Precision", "Recall", "F1", "Total", "FP");
intentTable.AddRow("*", allIntentsPrecision, allIntentsRecall, allIntentsF1, allIntentsTotal, allIntentsFP);
compareResults.Statistics.ByIntent
.OrderBy(intentItem => intentItem.Key)
.ToList()
.ForEach(intentItem =>
{
var intentBaseline = default(ConfusionMatrix);
if (baseline != null && !baseline.ByIntent.TryGetValue(intentItem.Key, out intentBaseline))
{
intentBaseline = ConfusionMatrix.Default;
}
var intentPrecision = Print(intentItem.Value.Precision(), intentBaseline?.Precision());
var intentRecall = Print(intentItem.Value.Recall(), intentBaseline?.Recall());
var intentF1 = Print(intentItem.Value.F1(), intentBaseline?.F1());
var intentTotal = Print(intentItem.Value.Total(), intentBaseline?.Total(), 0);
var intentFP = Print(intentItem.Value.FalsePositive, intentBaseline?.FalsePositive, 0);
intentTable.AddRow(intentItem.Key, intentPrecision, intentRecall, intentF1, intentTotal, intentFP);
});
intentTable.Write(Format.MarkDown);
var allEntitiesPrecision = Print(compareResults.Statistics.Entity.Precision(), baseline?.Entity.Precision());
var allEntitiesRecall = Print(compareResults.Statistics.Entity.Recall(), baseline?.Entity.Recall());
var allEntitiesF1 = Print(compareResults.Statistics.Entity.F1(), baseline?.Entity.F1());
var allEntitiesTotal = Print(compareResults.Statistics.Entity.Total(), baseline?.Entity.Total(), 0);
var allEntitiesFP = Print(compareResults.Statistics.Entity.FalsePositive, baseline?.Entity.FalsePositive, 0);
Console.WriteLine("## Entity Results");
var entityTable = new ConsoleTable("Entity", "Precision", "Recall", "F1", "Total", "FP");
entityTable.AddRow("*", allEntitiesPrecision, allEntitiesRecall, allEntitiesF1, allEntitiesTotal, allEntitiesFP);
compareResults.Statistics.ByEntityType
.OrderBy(entityItem => entityItem.Key)
.ToList()
.ForEach(entityItem =>
{
var entityBaseline = default(ConfusionMatrix);
if (baseline != null && !baseline.ByEntityType.TryGetValue(entityItem.Key, out entityBaseline))
{
entityBaseline = ConfusionMatrix.Default;
}
var entityPrecision = Print(entityItem.Value.Precision(), entityBaseline?.Precision());
var entityRecall = Print(entityItem.Value.Recall(), entityBaseline?.Recall());
var entityF1 = Print(entityItem.Value.F1(), entityBaseline?.F1());
var entityTotal = Print(entityItem.Value.Total(), entityBaseline?.Total(), 0);
var entityFP = Print(entityItem.Value.FalsePositive, entityBaseline?.FalsePositive, 0);
entityTable.AddRow(entityItem.Key, entityPrecision, entityRecall, entityF1, entityTotal, entityFP);
});
entityTable.Write(Format.MarkDown);
PrintIntentConfusionTable(compareResults.TestCases);
}