internal static IEnumerable <TestCase> ToIntentTestCases(LabeledUtterancePair pair) { var expectedUtterance = pair.Expected; var actualUtterance = pair.Actual; var score = actualUtterance.GetScore(); var text = expectedUtterance.Text; var expected = expectedUtterance.Intent ?? "null"; var actual = actualUtterance.Intent ?? "null"; if (expected == actual) { yield return(TruePositive( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, expected, new[] { expected, text }, "Utterances have matching intent.", "Intent")); } else { yield return(FalseNegative( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, expected, new[] { expected, actual, text }, $"Expected intent '{expected}', actual intent '{actual}'.", "Intent")); yield return(FalsePositive( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, actual, new[] { expected, actual, text }, $"Expected intent '{expected}', actual intent '{actual}'.", "Intent")); } }
internal static IEnumerable <TestCase> ToTextTestCases(LabeledUtterancePair pair) { var expectedUtterance = pair.Expected; // Skip if the test was not a speech test if (expectedUtterance.GetProperty <string>("speechFile") == null) { yield break; } var actualUtterance = pair.Actual; var expected = expectedUtterance.Text; var actual = actualUtterance.Text; var score = actualUtterance.GetTextScore(); if (expected == null && actual == null) { yield return(TrueNegative( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, Array.Empty <string>(), "Both utterances are 'null'.", "Text")); } else if (actual == null) { yield return(FalseNegative( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, new[] { expected }, $"Actual text is 'null', expected '{expected}'", "Text")); } else if (EqualsNormalized(expected, actual)) { yield return(TruePositive( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, new[] { expected }, "Utterances have matching text.", "Text")); } else { yield return(FalsePositive( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, new[] { expected, actual }, $"Expected text '{expected}', actual text '{actual}'.", "Text")); } }
internal static IEnumerable <TestCase> ToEntityTestCases(LabeledUtterancePair pair) { var expectedUtterance = pair.Expected; var actualUtterance = pair.Actual; var text = expectedUtterance.Text; var expected = expectedUtterance.Entities; var actual = actualUtterance.Entities; if ((expected == null || expected.Count == 0) && (actual == null || actual.Count == 0)) { yield return(TrueNegative( pair.UtteranceId, ComparisonTargetKind.Entity, expectedUtterance, actualUtterance, 0, null, new[] { text }, "Neither utterances have entities.", "Entity")); yield break; } bool isEntityMatch(Entity expectedEntity, Entity actualEntity) { return(expectedEntity.EntityType == actualEntity.EntityType && (isEntityTextMatch(expectedEntity, actualEntity) || isEntityValueMatch(expectedEntity, actualEntity))); } bool isEntityTextMatch(Entity expectedEntity, Entity actualEntity) { return(expectedEntity.MatchText != null && EqualsNormalized(expectedEntity.MatchText, actualEntity.MatchText) && expectedEntity.MatchIndex == actualEntity.MatchIndex); } bool isEntityValueMatch(Entity expectedEntity, Entity actualEntity) { /* Required case to support NLU providers that do not specify matched text */ return(actualEntity.MatchText == null && (EqualsNormalizedJson(expectedEntity.EntityValue, actualEntity.EntityValue) || EqualsNormalizedJson(expectedEntity.MatchText, actualEntity.EntityValue))); } if (expected != null) { foreach (var entity in expected) { // "Soft" entity match test cases var entityValue = entity.MatchText ?? entity.EntityValue; var formattedEntity = entityValue.ToString(Formatting.None); var matchedEntity = actual != null ? actual.FirstOrDefault(actualEntity => isEntityMatch(entity, actualEntity)) : null; var score = matchedEntity.GetScore(); if (matchedEntity == null) { yield return(FalseNegative( pair.UtteranceId, ComparisonTargetKind.Entity, expectedUtterance, actualUtterance, score, entity.EntityType, new[] { entity.EntityType, formattedEntity, text }, $"Actual utterance does not have entity matching '{entityValue}'.", "Entity")); } else { yield return(TruePositive( pair.UtteranceId, ComparisonTargetKind.Entity, expectedUtterance, actualUtterance, score, entity.EntityType, new[] { entity.EntityType, formattedEntity, text }, $"Both utterances have entity '{entityValue}'.", "Entity")); } if (matchedEntity != null) { if (entity.EntityValue != null && entity.EntityValue.Type != JTokenType.Null) { var formattedEntityValue = entity.EntityValue.ToString(Formatting.None); if (!ContainsSubtree(entity.EntityValue, matchedEntity.EntityValue)) { yield return(FalseNegative( pair.UtteranceId, ComparisonTargetKind.EntityValue, expectedUtterance, actualUtterance, score, entity.EntityType, new[] { entity.EntityType, formattedEntityValue, text }, $"Actual utterance does not have entity value matching '{formattedEntityValue}'.", "Entity")); } else { yield return(TruePositive( pair.UtteranceId, ComparisonTargetKind.EntityValue, expectedUtterance, actualUtterance, score, entity.EntityType, new[] { entity.EntityType, formattedEntityValue, text }, $"Both utterances have entity value '{formattedEntityValue}'.", "Entity")); } } } } } if (actual != null) { foreach (var entity in actual) { var score = entity.GetScore(); var entityValue = entity.MatchText ?? entity.EntityValue; if (expected == null || !expected.Any(expectedEntity => isEntityMatch(expectedEntity, entity))) { yield return(FalsePositive( pair.UtteranceId, ComparisonTargetKind.Entity, expectedUtterance, actualUtterance, score, entity.EntityType, new[] { entity.EntityType, entityValue.ToString(Formatting.None), text }, $"Expected utterance does not have entity matching '{entityValue}'.", "Entity")); } } } }
internal static IEnumerable <TestCase> ToIntentTestCases(LabeledUtterancePair pair) { var expectedUtterance = pair.Expected; var actualUtterance = pair.Actual; var score = actualUtterance.GetScore(); var text = expectedUtterance.Text; var expected = expectedUtterance.Intent; var actual = actualUtterance.Intent; bool isNoneIntent(string intent) { return(intent == null || intent == "None"); } if (isNoneIntent(expected) && isNoneIntent(actual)) { yield return(TrueNegative( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, null, new[] { text }, "Both intents are 'None'.", "Intent")); yield break; } if (expected == actual) { yield return(TruePositive( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, expected, new[] { expected, text }, "Utterances have matching intent.", "Intent")); yield break; } if (!isNoneIntent(expected)) { yield return(FalseNegative( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, expected, new[] { expected, actual, text }, $"Expected intent '{expected}', actual intent '{actual}'.", "Intent")); } if (!isNoneIntent(actual)) { yield return(FalsePositive( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, actual, new[] { expected, actual, text }, $"Expected intent '{expected}', actual intent '{actual}'.", "Intent")); } }
internal static TestCase ToTextTestCase(LabeledUtterancePair pair) { var expectedUtterance = pair.Expected; var actualUtterance = pair.Actual; var expected = expectedUtterance.Text; var actual = actualUtterance.Text; var score = actualUtterance.GetTextScore(); if (expected == null && actual == null) { return(TrueNegative( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, Array.Empty <string>(), "Both utterances are 'null'.", "Text")); } if (actual == null) { return(FalseNegative( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, new[] { expected }, $"Actual text is 'null', expected '{expected}'", "Text")); } if (EqualsNormalized(expected, actual)) { return(TruePositive( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, new[] { expected }, "Utterances have matching text.", "Text")); } return(FalsePositive( pair.UtteranceId, ComparisonTargetKind.Text, expectedUtterance, actualUtterance, score, null, new[] { expected, actual }, $"Expected text '{expected}', actual text '{actual}'.", "Text")); }
internal static TestCase ToIntentTestCase(LabeledUtterancePair pair) { var expectedUtterance = pair.Expected; var actualUtterance = pair.Actual; var score = actualUtterance is ScoredLabeledUtterance scoredUtterance ? scoredUtterance.Score : 0; var text = expectedUtterance.Text; var expected = expectedUtterance.Intent; var actual = actualUtterance.Intent; bool isNoneIntent(string intent) { return(intent == null || intent == "None"); } if (isNoneIntent(actual)) { return(isNoneIntent(expected) ? TrueNegative( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, null, new[] { text }, "Both intents are 'None'.", "Intent") : FalseNegative( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, expected, new[] { expected, text }, $"Actual intent is 'None', expected '{expected}'", "Intent")); } if (expected == actual) { return(TruePositive( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, expected, new[] { expected, text }, "Utterances have matching intent.", "Intent")); } return(FalsePositive( pair.UtteranceId, ComparisonTargetKind.Intent, expectedUtterance, actualUtterance, score, isNoneIntent(expected) ? actual : expected, new[] { expected, actual, text }, $"Expected intent '{expected}', actual intent '{actual}'.", "Intent")); }