internal static IEnumerable <TestCase> ToEntityTestCases(LabeledUtteranceTestInput testInput)
{
var expectedUtterance = testInput.Expected;
var actualUtterance = testInput.Actual;
var text = expectedUtterance.Text;
var expected = expectedUtterance.Entities;
var actual = actualUtterance.Entities;
if ((expected == null || expected.Count == 0) && (actual == null || actual.Count == 0))
{
yield return(TrueNegative(
testInput.UtteranceId,
ComparisonTargetKind.Entity,
expectedUtterance,
actualUtterance,
0,
null,
new[] { text },
"Neither utterances have entities.",
"Entity"));
yield break;
}
bool isEntityMatch(IEntity expectedEntity, IEntity actualEntity)
{
return(expectedEntity.EntityType == actualEntity.EntityType &&
(isEntityTextMatch(expectedEntity, actualEntity) ||
isEntityValueMatch(expectedEntity, actualEntity)));
}
bool isEntityTextMatch(IEntity expectedEntity, IEntity actualEntity)
{
return(expectedEntity.MatchText != null &&
EqualsNormalized(expectedEntity.MatchText, actualEntity.MatchText) &&
expectedEntity.MatchIndex == actualEntity.MatchIndex);
}
bool isEntityValueMatch(IEntity expectedEntity, IEntity actualEntity)
{
/* Required case to support NLU providers that do not specify matched text */
return(actualEntity.MatchText == null &&
(EqualsNormalizedJson(expectedEntity.EntityValue, actualEntity.EntityValue) ||
EqualsNormalizedJson(expectedEntity.MatchText, actualEntity.EntityValue)));
}
if (expected != null)
{
foreach (var entity in expected)
{
// "Soft" entity match test cases
var entityValue = entity.MatchText ?? entity.EntityValue;
var formattedEntity = entityValue.ToString(Formatting.None);
var matchedEntity = actual != null
? actual.FirstOrDefault(actualEntity => isEntityMatch(entity, actualEntity))
: null;
var score = matchedEntity.GetScore();
if (matchedEntity == null)
{
yield return(FalseNegative(
testInput.UtteranceId,
ComparisonTargetKind.Entity,
expectedUtterance,
actualUtterance,
score,
entity.EntityType,
new[] { entity.EntityType, formattedEntity, text },
$"Actual utterance does not have entity matching '{entityValue}'.",
"Entity"));
}
else
{
yield return(TruePositive(
testInput.UtteranceId,
ComparisonTargetKind.Entity,
expectedUtterance,
actualUtterance,
score,
entity.EntityType,
new[] { entity.EntityType, formattedEntity, text },
$"Both utterances have entity '{entityValue}'.",
"Entity"));
}
if (matchedEntity != null)
{
if (entity.EntityValue != null && entity.EntityValue.Type != JTokenType.Null)
{
var formattedEntityValue = entity.EntityValue.ToString(Formatting.None);
if (!ContainsSubtree(entity.EntityValue, matchedEntity.EntityValue))
{
yield return(FalseNegative(
testInput.UtteranceId,
ComparisonTargetKind.EntityValue,
expectedUtterance,
actualUtterance,
score,
entity.EntityType,
new[] { entity.EntityType, formattedEntityValue, text },
$"Actual utterance does not have entity value matching '{formattedEntityValue}'.",
"Entity"));
}
else
{
yield return(TruePositive(
testInput.UtteranceId,
ComparisonTargetKind.EntityValue,
expectedUtterance,
actualUtterance,
score,
entity.EntityType,
new[] { entity.EntityType, formattedEntityValue, text },
$"Both utterances have entity value '{formattedEntityValue}'.",
"Entity"));
}
}
}
}
}
if (actual != null)
{
foreach (var entity in actual)
{
var score = entity.GetScore();
var entityValue = entity.MatchText ?? entity.EntityValue;
var isStrictEntity = IsStrictEntity(entity.EntityType, expectedUtterance, testInput.TestSettings);
if (isStrictEntity && (expected == null || !expected.Any(expectedEntity => isEntityMatch(expectedEntity, entity))))
{
yield return(FalsePositive(
testInput.UtteranceId,
ComparisonTargetKind.Entity,
expectedUtterance,
actualUtterance,
score,
entity.EntityType,
new[] { entity.EntityType, entityValue.ToString(Formatting.None), text },
$"Expected utterance does not have entity matching '{entityValue}'.",
"Entity"));
}
}
}
}
internal static IEnumerable <TestCase> ToTextTestCases(LabeledUtteranceTestInput testInput)
{
var expectedUtterance = testInput.Expected;
// Skip if the test was not a speech test
if (expectedUtterance.GetProperty <string>("speechFile") == null)
{
yield break;
}
var actualUtterance = testInput.Actual;
var expected = expectedUtterance.Text;
var actual = actualUtterance.Text;
var score = actualUtterance.GetTextScore();
if (expected == null && actual == null)
{
yield return(TrueNegative(
testInput.UtteranceId,
ComparisonTargetKind.Text,
expectedUtterance,
actualUtterance,
score,
null,
Array.Empty <string>(),
"Both utterances are 'null'.",
"Text"));
}
else if (actual == null)
{
yield return(FalseNegative(
testInput.UtteranceId,
ComparisonTargetKind.Text,
expectedUtterance,
actualUtterance,
score,
null,
new[] { expected },
$"Actual text is 'null', expected '{expected}'",
"Text"));
}
else if (EqualsNormalized(expected, actual))
{
yield return(TruePositive(
testInput.UtteranceId,
ComparisonTargetKind.Text,
expectedUtterance,
actualUtterance,
score,
null,
new[] { expected },
"Utterances have matching text.",
"Text"));
}
else if (!testInput.TestSettings.UnitTestMode)
{
yield return(FalsePositive(
testInput.UtteranceId,
ComparisonTargetKind.Text,
expectedUtterance,
actualUtterance,
score,
null,
new[] { expected, actual },
$"Expected text '{expected}', actual text '{actual}'.",
"Text"));
}
}
internal static IEnumerable <TestCase> ToIntentTestCases(LabeledUtteranceTestInput testInput)
{
var expectedUtterance = testInput.Expected;
var actualUtterance = testInput.Actual;
var score = actualUtterance.GetScore();
var text = expectedUtterance.Text;
var expected = expectedUtterance.Intent;
var actual = actualUtterance.Intent;
bool isNoneIntent(string intent)
{
return(intent == null || intent == testInput.TestSettings.TrueNegativeIntent);
}
if (isNoneIntent(expected) && isNoneIntent(actual))
{
yield return(TrueNegative(
testInput.UtteranceId,
ComparisonTargetKind.Intent,
expectedUtterance,
actualUtterance,
score,
null,
new[] { text },
"Both intents are 'None'.",
"Intent"));
yield break;
}
if (expected == actual)
{
yield return(TruePositive(
testInput.UtteranceId,
ComparisonTargetKind.Intent,
expectedUtterance,
actualUtterance,
score,
expected,
new[] { expected, text },
"Utterances have matching intent.",
"Intent"));
yield break;
}
if (!isNoneIntent(expected))
{
yield return(FalseNegative(
testInput.UtteranceId,
ComparisonTargetKind.Intent,
expectedUtterance,
actualUtterance,
score,
expected,
new[] { expected, actual, text },
$"Expected intent '{expected}', actual intent '{actual}'.",
"Intent"));
}
if (!isNoneIntent(actual) && !testInput.TestSettings.UnitTestMode)
{
yield return(FalsePositive(
testInput.UtteranceId,
ComparisonTargetKind.Intent,
expectedUtterance,
actualUtterance,
score,
actual,
new[] { expected, actual, text },
$"Expected intent '{expected}', actual intent '{actual}'.",
"Intent"));
}
}
