/// <summary> /// Checks that NLU results do not regress performance against the baseline beyond the given thresholds. /// </summary> /// <param name="currentResults">Current results.</param> /// <param name="baselineResults">Baseline results.</param> /// <param name="threshold">Performance threshold.</param> /// <returns> /// <code>true</code> if performance has not dropped beyond thresholds, otherwise <code>false</code>. /// </returns> public static bool CheckThreshold(this NLUStatistics currentResults, NLUStatistics baselineResults, NLUThreshold threshold) { ConfusionMatrix getResultsByGroup(string group, IReadOnlyDictionary <string, ConfusionMatrix> groupStatistics) { if (groupStatistics != null && groupStatistics.TryGetValue(group, out var result)) { return(result); } return(null); } ConfusionMatrix getResults(NLUStatistics statistics) { if (threshold.Type == "intent" || threshold.Type == "entity") { if (threshold.Group == null || threshold.Group == "*") { return(threshold.Type == "intent" ? statistics?.Intent : statistics?.Entity); } return(threshold.Type == "intent" ? getResultsByGroup(threshold.Group, statistics?.ByIntent) : getResultsByGroup(threshold.Group, statistics?.ByEntityType)); } return(null); } var currentMatrix = getResults(currentResults) ?? ConfusionMatrix.Default; if (threshold.Comparison == NLUThresholdKind.Absolute) { return(currentMatrix.GetMetric(threshold.Metric) >= threshold.Threshold); } var baselineMatrix = getResults(baselineResults) ?? ConfusionMatrix.Default; return(baselineMatrix.GetMetric(threshold.Metric) - currentMatrix.GetMetric(threshold.Metric) <= threshold.Threshold); }
/// <summary> /// Prints to the console the intents, entities performance results /// and a confusion table for intents /// </summary> /// <param name="compareResults"> The comparison results for intents and entities</param> /// <param name="baseline">The baseline results the results are benchmarked against.</param> public static void PrintResults(this NLUCompareResults compareResults, NLUStatistics baseline) { var allIntentsPrecision = Print(compareResults.Statistics.Intent.Precision(), baseline?.Intent.Precision()); var allIntentsRecall = Print(compareResults.Statistics.Intent.Recall(), baseline?.Intent.Recall()); var allIntentsF1 = Print(compareResults.Statistics.Intent.F1(), baseline?.Intent.F1()); var allIntentsTotal = Print(compareResults.Statistics.Intent.Total(), baseline?.Intent.Total(), 0); var allIntentsFP = Print(compareResults.Statistics.Intent.FalsePositive, baseline?.Intent.FalsePositive, 0); Console.WriteLine("## Intents Results"); var intentTable = new ConsoleTable("Intent", "Precision", "Recall", "F1", "Total", "FP"); intentTable.AddRow("*", allIntentsPrecision, allIntentsRecall, allIntentsF1, allIntentsTotal, allIntentsFP); compareResults.Statistics.ByIntent .OrderBy(intentItem => intentItem.Key) .ToList() .ForEach(intentItem => { var intentBaseline = default(ConfusionMatrix); if (baseline != null && !baseline.ByIntent.TryGetValue(intentItem.Key, out intentBaseline)) { intentBaseline = ConfusionMatrix.Default; } var intentPrecision = Print(intentItem.Value.Precision(), intentBaseline?.Precision()); var intentRecall = Print(intentItem.Value.Recall(), intentBaseline?.Recall()); var intentF1 = Print(intentItem.Value.F1(), intentBaseline?.F1()); var intentTotal = Print(intentItem.Value.Total(), intentBaseline?.Total(), 0); var intentFP = Print(intentItem.Value.FalsePositive, intentBaseline?.FalsePositive, 0); intentTable.AddRow(intentItem.Key, intentPrecision, intentRecall, intentF1, intentTotal, intentFP); }); intentTable.Write(Format.MarkDown); var allEntitiesPrecision = Print(compareResults.Statistics.Entity.Precision(), baseline?.Entity.Precision()); var allEntitiesRecall = Print(compareResults.Statistics.Entity.Recall(), baseline?.Entity.Recall()); var allEntitiesF1 = Print(compareResults.Statistics.Entity.F1(), baseline?.Entity.F1()); var allEntitiesTotal = Print(compareResults.Statistics.Entity.Total(), baseline?.Entity.Total(), 0); var allEntitiesFP = Print(compareResults.Statistics.Entity.FalsePositive, baseline?.Entity.FalsePositive, 0); Console.WriteLine("## Entity Results"); var entityTable = new ConsoleTable("Entity", "Precision", "Recall", "F1", "Total", "FP"); entityTable.AddRow("*", allEntitiesPrecision, allEntitiesRecall, allEntitiesF1, allEntitiesTotal, allEntitiesFP); compareResults.Statistics.ByEntityType .OrderBy(entityItem => entityItem.Key) .ToList() .ForEach(entityItem => { var entityBaseline = default(ConfusionMatrix); if (baseline != null && !baseline.ByEntityType.TryGetValue(entityItem.Key, out entityBaseline)) { entityBaseline = ConfusionMatrix.Default; } var entityPrecision = Print(entityItem.Value.Precision(), entityBaseline?.Precision()); var entityRecall = Print(entityItem.Value.Recall(), entityBaseline?.Recall()); var entityF1 = Print(entityItem.Value.F1(), entityBaseline?.F1()); var entityTotal = Print(entityItem.Value.Total(), entityBaseline?.Total(), 0); var entityFP = Print(entityItem.Value.FalsePositive, entityBaseline?.FalsePositive, 0); entityTable.AddRow(entityItem.Key, entityPrecision, entityRecall, entityF1, entityTotal, entityFP); }); entityTable.Write(Format.MarkDown); PrintIntentConfusionTable(compareResults.TestCases); }