public static void GetNLUCompareResultsEntityValueStatistics(
string expected,
string actual,
int truePositive,
int trueNegative,
int falsePositive,
int falseNegative)
{
var entityType = Guid.NewGuid().ToString();
var matchText = Guid.NewGuid().ToString();
var expectedJson = expected == null?JValue.CreateNull() : (JValue)expected;
var actualJson = actual == null?JValue.CreateNull() : (JValue)actual;
var expectedEntity = new[] { new Entity(entityType, expectedJson, matchText, 0) };
var actualEntity = new[] { new Entity(entityType, actualJson, matchText, 0) };
var expectedUtterance = new LabeledUtterance(null, null, expectedEntity);
var actualUtterance = new LabeledUtterance(null, null, actualEntity);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
GetDefaultTestSettings());
compareResults.Statistics.EntityValue.TruePositive.Should().Be(truePositive);
compareResults.Statistics.EntityValue.TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.EntityValue.FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.EntityValue.FalseNegative.Should().Be(falseNegative);
if (expected != null)
{
compareResults.Statistics.ByEntityValueType[entityType].TruePositive.Should().Be(truePositive);
compareResults.Statistics.ByEntityValueType[entityType].TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.ByEntityValueType[entityType].FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.ByEntityValueType[entityType].FalseNegative.Should().Be(falseNegative);
}
}
public static void GetNLUCompareResultsFalsePositiveEntityDifferentType()
{
var expectedEntityType = Guid.NewGuid().ToString();
var actualEntityType = Guid.NewGuid().ToString();
var matchText = Guid.NewGuid().ToString();
var expectedEntity = new[] { new Entity(expectedEntityType, null, matchText, 0) };
var actualEntity = new[] { new Entity(actualEntityType, null, matchText, 0) };
var expectedUtterance = new LabeledUtterance(null, null, expectedEntity);
var actualUtterance = new LabeledUtterance(null, null, actualEntity);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
GetDefaultTestSettings());
compareResults.Statistics.Entity.TruePositive.Should().Be(0);
compareResults.Statistics.Entity.TrueNegative.Should().Be(0);
compareResults.Statistics.Entity.FalsePositive.Should().Be(1);
compareResults.Statistics.Entity.FalseNegative.Should().Be(1);
compareResults.Statistics.ByEntityType[expectedEntityType].TruePositive.Should().Be(0);
compareResults.Statistics.ByEntityType[expectedEntityType].TrueNegative.Should().Be(0);
compareResults.Statistics.ByEntityType[expectedEntityType].FalsePositive.Should().Be(0);
compareResults.Statistics.ByEntityType[expectedEntityType].FalseNegative.Should().Be(1);
compareResults.Statistics.ByEntityType[actualEntityType].TruePositive.Should().Be(0);
compareResults.Statistics.ByEntityType[actualEntityType].TrueNegative.Should().Be(0);
compareResults.Statistics.ByEntityType[actualEntityType].FalsePositive.Should().Be(1);
compareResults.Statistics.ByEntityType[actualEntityType].FalseNegative.Should().Be(0);
}
public static void GetNLUCompareResultsIntentStatistics(
string expected,
string actual,
int truePositive,
int trueNegative,
int falsePositive,
int falseNegative)
{
var expectedUtterance = new LabeledUtterance(null, expected, null);
var actualUtterance = new LabeledUtterance(null, actual, null);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
GetDefaultTestSettings());
compareResults.Statistics.Intent.TruePositive.Should().Be(truePositive);
compareResults.Statistics.Intent.TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.Intent.FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.Intent.FalseNegative.Should().Be(falseNegative);
if (expected != null && expected != "None")
{
compareResults.Statistics.ByIntent[expected].TruePositive.Should().Be(truePositive);
compareResults.Statistics.ByIntent[expected].FalseNegative.Should().Be(falseNegative);
}
else if (actual != null && actual != "None")
{
compareResults.Statistics.ByIntent[actual].FalsePositive.Should().Be(falsePositive);
}
}
public static void GetNLUCompareResultsEntityStatistics(
string expected,
string actual,
int truePositive,
int trueNegative,
int falsePositive,
int falseNegative)
{
var entityType = Guid.NewGuid().ToString();
var expectedEntity = expected != null ? new[] { new Entity(entityType, null, expected, 0) } : null;
var actualEntity = actual != null ? new[] { new Entity(entityType, null, actual, 0) } : null;
var expectedUtterance = new LabeledUtterance(null, null, expectedEntity);
var actualUtterance = new LabeledUtterance(null, null, actualEntity);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
GetDefaultTestSettings());
compareResults.Statistics.Entity.TruePositive.Should().Be(truePositive);
compareResults.Statistics.Entity.TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.Entity.FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.Entity.FalseNegative.Should().Be(falseNegative);
if (expected != null || actual != null)
{
compareResults.Statistics.ByEntityType[entityType].TruePositive.Should().Be(truePositive);
compareResults.Statistics.ByEntityType[entityType].TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.ByEntityType[entityType].FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.ByEntityType[entityType].FalseNegative.Should().Be(falseNegative);
}
}
public static void GetNLUCompareResultsIntentStatistics(
string expected,
string actual,
int truePositive,
int trueNegative,
int falsePositive,
int falseNegative)
{
var expectedUtterance = new LabeledUtterance(null, expected, null);
var actualUtterance = new LabeledUtterance(null, actual, null);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance });
compareResults.Statistics.Intent.TruePositive.Should().Be(truePositive);
compareResults.Statistics.Intent.TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.Intent.FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.Intent.FalseNegative.Should().Be(falseNegative);
actual = actual ?? "null";
expected = expected ?? "null";
compareResults.Statistics.ByIntent[expected].TruePositive.Should().Be(truePositive);
compareResults.Statistics.ByIntent[expected].FalseNegative.Should().Be(falseNegative);
compareResults.Statistics.ByIntent[actual].FalsePositive.Should().Be(falsePositive);
}
public static void UsesIndexAsUtteranceId()
{
var expectedUtterance = new LabeledUtterance(null, "Greeting", null);
var actualUtterance = new LabeledUtterance(null, "Greeting", null);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance, expectedUtterance },
new[] { actualUtterance, actualUtterance });
compareResults.TestCases.Count.Should().Be(4);
compareResults.TestCases.Where(t => t.UtteranceId == "0").Count().Should().Be(2);
compareResults.TestCases.Where(t => t.UtteranceId == "1").Count().Should().Be(2);
}
public static void GlobalLocalStrictIgnoreEntities(
string expected,
string actual,
string value,
bool unitTestMode,
int truePositive,
int trueNegative,
int falsePositive,
int falseNegative)
{
var globalSettingsFile = Guid.NewGuid().ToString();
var globalSettings = new JObject
{
{ "strictEntities", new JArray {
"c", "e"
} },
{ "ignoreEntities", new JArray {
"d", "f"
} },
};
File.WriteAllText(globalSettingsFile, globalSettings.ToString());
try
{
var expectedEntity = new Entity(expected, null, string.Empty, 0);
var actualEntity = new Entity(actual, null, value, 0);
var expectedUtterance = new LabeledUtterance(null, null, new[] { expectedEntity })
.WithProperty("strictEntities", new JArray {
"a", "f"
})
.WithProperty("ignoreEntities", new JArray {
"b", "e"
});
var actualUtterance = new LabeledUtterance(null, null, new[] { actualEntity });
var testSettings = new TestSettings(globalSettingsFile, unitTestMode);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
testSettings);
compareResults.Statistics.Entity.TruePositive.Should().Be(truePositive);
compareResults.Statistics.Entity.TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.Entity.FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.Entity.FalseNegative.Should().Be(falseNegative);
}
finally
{
File.Delete(globalSettingsFile);
}
}
public static void UsesInputUtteranceId()
{
var utteranceId = Guid.NewGuid().ToString();
var expectedUtterance = new CompareLabeledUtterance(utteranceId, null, "Greeting", null);
var actualUtterance = new LabeledUtterance(null, "Greeting", null);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
false);
compareResults.TestCases.Count.Should().Be(2);
compareResults.TestCases.Where(t => t.UtteranceId == utteranceId).Count().Should().Be(2);
}
public static void UsesInputUtteranceId()
{
var utteranceId = Guid.NewGuid().ToString();
var expectedUtterance = new JsonLabeledUtterance(null, "Greeting", null);
expectedUtterance.AdditionalProperties.Add("utteranceId", utteranceId);
var actualUtterance = new LabeledUtterance(null, "Greeting", null);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance });
compareResults.TestCases.Count.Should().Be(2);
compareResults.TestCases.Where(t => t.UtteranceId == utteranceId).Count().Should().Be(2);
}
public static void GetNLUCompareResultsNullScores()
{
var entityType = Guid.NewGuid().ToString();
var matchText = Guid.NewGuid().ToString();
var expectedEntity = new[] { new Entity(entityType, null, matchText, 0) };
var actualEntity = new[] { new Entity(entityType, null, matchText, 0) };
var expectedUtterance = new LabeledUtterance(null, null, expectedEntity);
var actualUtterance = new LabeledUtterance(null, null, actualEntity);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance });
compareResults.TestCases.Select(t => t.Score).Any(score => score != null).Should().BeFalse();
}
public static void GetNLUCompareResultsExtractsFalsePositiveEntityScore()
{
var entityType = Guid.NewGuid().ToString();
var matchText = Guid.NewGuid().ToString();
var actualEntity = new[] { new Entity(entityType, null, matchText, 0).WithScore(0.5) };
var expectedUtterance = new LabeledUtterance(null, null, null);
var actualUtterance = new LabeledUtterance(null, null, actualEntity);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance });
var testCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Entity);
testCase.Should().NotBeNull();
testCase.Score.Should().Be(0.5);
}
public static void GetNLUCompareResultsDefaultScores()
{
var entityType = Guid.NewGuid().ToString();
var matchText = Guid.NewGuid().ToString();
var expectedEntity = new[] { new Entity(entityType, null, matchText, 0) };
var actualEntity = new[] { new Entity(entityType, null, matchText, 0) };
var expectedUtterance = new LabeledUtterance(null, null, expectedEntity);
var actualUtterance = new LabeledUtterance(null, null, actualEntity);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
false);
compareResults.TestCases.Select(t => t.Score).Should().AllBeEquivalentTo(0);
}
public static void NoFalsePositiveIntentsUnitTestMode()
{
var expectedUtterance = new JsonLabeledUtterance(new JsonEntities(Array.Empty <Entity>()));
var actualUtterance = new LabeledUtterance(null, "foo", null);
var testSettings = new TestSettings(default(string), true);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
testSettings);
compareResults.Statistics.Intent.TruePositive.Should().Be(0);
compareResults.Statistics.Intent.TrueNegative.Should().Be(0);
compareResults.Statistics.Intent.FalsePositive.Should().Be(0);
compareResults.Statistics.Intent.FalseNegative.Should().Be(0);
}
public static void GetNLUCompareResultsExtractsIntentAndTextScore()
{
var expectedUtterance = new LabeledUtterance(null, null, null).WithProperty("speechFile", "speechFile");
var actualUtterance = new LabeledUtterance(null, null, null).WithScore(0.5).WithTextScore(0.1);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance });
var intentTestCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Intent);
intentTestCase.Should().NotBeNull();
intentTestCase.Score.Should().Be(0.5);
var textTestCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Text);
textTestCase.Should().NotBeNull();
textTestCase.Score.Should().Be(0.1);
}
public static void GetNLUCompareResultsTextStatistics(
string expected,
string actual,
int truePositive,
int trueNegative,
int falsePositive,
int falseNegative)
{
var expectedUtterance = new LabeledUtterance(expected, null, null).WithProperty("speechFile", "speechFile");
var actualUtterance = new LabeledUtterance(actual, null, null);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance });
compareResults.Statistics.Text.TruePositive.Should().Be(truePositive);
compareResults.Statistics.Text.TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.Text.FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.Text.FalseNegative.Should().Be(falseNegative);
}
public static void CustomTrueNegativeIntent(
string expected,
string actual,
int truePositive,
int trueNegative,
int falsePositive,
int falseNegative)
{
var expectedUtterance = new LabeledUtterance(null, expected, null);
var actualUtterance = new LabeledUtterance(null, actual, null);
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary <string, string>
{
{ "trueNegativeIntent", "foo" },
})
.Build();
var testSettings = new TestSettings(configuration, false);
var compareResults = TestCaseSource.GetNLUCompareResults(
new[] { expectedUtterance },
new[] { actualUtterance },
testSettings);
compareResults.Statistics.Intent.TruePositive.Should().Be(truePositive);
compareResults.Statistics.Intent.TrueNegative.Should().Be(trueNegative);
compareResults.Statistics.Intent.FalsePositive.Should().Be(falsePositive);
compareResults.Statistics.Intent.FalseNegative.Should().Be(falseNegative);
if (expected != null && expected != "foo")
{
compareResults.Statistics.ByIntent[expected].TruePositive.Should().Be(truePositive);
compareResults.Statistics.ByIntent[expected].FalseNegative.Should().Be(falseNegative);
}
else if (actual != null && actual != "foo")
{
compareResults.Statistics.ByIntent[actual].FalsePositive.Should().Be(falsePositive);
}
}