public static void GetNLUCompareResultsEntityValueStatistics( string expected, string actual, int truePositive, int trueNegative, int falsePositive, int falseNegative) { var entityType = Guid.NewGuid().ToString(); var matchText = Guid.NewGuid().ToString(); var expectedJson = expected == null?JValue.CreateNull() : (JValue)expected; var actualJson = actual == null?JValue.CreateNull() : (JValue)actual; var expectedEntity = new[] { new Entity(entityType, expectedJson, matchText, 0) }; var actualEntity = new[] { new Entity(entityType, actualJson, matchText, 0) }; var expectedUtterance = new LabeledUtterance(null, null, expectedEntity); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, GetDefaultTestSettings()); compareResults.Statistics.EntityValue.TruePositive.Should().Be(truePositive); compareResults.Statistics.EntityValue.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.EntityValue.FalsePositive.Should().Be(falsePositive); compareResults.Statistics.EntityValue.FalseNegative.Should().Be(falseNegative); if (expected != null) { compareResults.Statistics.ByEntityValueType[entityType].TruePositive.Should().Be(truePositive); compareResults.Statistics.ByEntityValueType[entityType].TrueNegative.Should().Be(trueNegative); compareResults.Statistics.ByEntityValueType[entityType].FalsePositive.Should().Be(falsePositive); compareResults.Statistics.ByEntityValueType[entityType].FalseNegative.Should().Be(falseNegative); } }
public static void GetNLUCompareResultsFalsePositiveEntityDifferentType() { var expectedEntityType = Guid.NewGuid().ToString(); var actualEntityType = Guid.NewGuid().ToString(); var matchText = Guid.NewGuid().ToString(); var expectedEntity = new[] { new Entity(expectedEntityType, null, matchText, 0) }; var actualEntity = new[] { new Entity(actualEntityType, null, matchText, 0) }; var expectedUtterance = new LabeledUtterance(null, null, expectedEntity); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, GetDefaultTestSettings()); compareResults.Statistics.Entity.TruePositive.Should().Be(0); compareResults.Statistics.Entity.TrueNegative.Should().Be(0); compareResults.Statistics.Entity.FalsePositive.Should().Be(1); compareResults.Statistics.Entity.FalseNegative.Should().Be(1); compareResults.Statistics.ByEntityType[expectedEntityType].TruePositive.Should().Be(0); compareResults.Statistics.ByEntityType[expectedEntityType].TrueNegative.Should().Be(0); compareResults.Statistics.ByEntityType[expectedEntityType].FalsePositive.Should().Be(0); compareResults.Statistics.ByEntityType[expectedEntityType].FalseNegative.Should().Be(1); compareResults.Statistics.ByEntityType[actualEntityType].TruePositive.Should().Be(0); compareResults.Statistics.ByEntityType[actualEntityType].TrueNegative.Should().Be(0); compareResults.Statistics.ByEntityType[actualEntityType].FalsePositive.Should().Be(1); compareResults.Statistics.ByEntityType[actualEntityType].FalseNegative.Should().Be(0); }
public static void GetNLUCompareResultsIntentStatistics( string expected, string actual, int truePositive, int trueNegative, int falsePositive, int falseNegative) { var expectedUtterance = new LabeledUtterance(null, expected, null); var actualUtterance = new LabeledUtterance(null, actual, null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, GetDefaultTestSettings()); compareResults.Statistics.Intent.TruePositive.Should().Be(truePositive); compareResults.Statistics.Intent.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Intent.FalsePositive.Should().Be(falsePositive); compareResults.Statistics.Intent.FalseNegative.Should().Be(falseNegative); if (expected != null && expected != "None") { compareResults.Statistics.ByIntent[expected].TruePositive.Should().Be(truePositive); compareResults.Statistics.ByIntent[expected].FalseNegative.Should().Be(falseNegative); } else if (actual != null && actual != "None") { compareResults.Statistics.ByIntent[actual].FalsePositive.Should().Be(falsePositive); } }
public static void GetNLUCompareResultsEntityStatistics( string expected, string actual, int truePositive, int trueNegative, int falsePositive, int falseNegative) { var entityType = Guid.NewGuid().ToString(); var expectedEntity = expected != null ? new[] { new Entity(entityType, null, expected, 0) } : null; var actualEntity = actual != null ? new[] { new Entity(entityType, null, actual, 0) } : null; var expectedUtterance = new LabeledUtterance(null, null, expectedEntity); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, GetDefaultTestSettings()); compareResults.Statistics.Entity.TruePositive.Should().Be(truePositive); compareResults.Statistics.Entity.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Entity.FalsePositive.Should().Be(falsePositive); compareResults.Statistics.Entity.FalseNegative.Should().Be(falseNegative); if (expected != null || actual != null) { compareResults.Statistics.ByEntityType[entityType].TruePositive.Should().Be(truePositive); compareResults.Statistics.ByEntityType[entityType].TrueNegative.Should().Be(trueNegative); compareResults.Statistics.ByEntityType[entityType].FalsePositive.Should().Be(falsePositive); compareResults.Statistics.ByEntityType[entityType].FalseNegative.Should().Be(falseNegative); } }
public static void GetNLUCompareResultsIntentStatistics( string expected, string actual, int truePositive, int trueNegative, int falsePositive, int falseNegative) { var expectedUtterance = new LabeledUtterance(null, expected, null); var actualUtterance = new LabeledUtterance(null, actual, null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }); compareResults.Statistics.Intent.TruePositive.Should().Be(truePositive); compareResults.Statistics.Intent.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Intent.FalsePositive.Should().Be(falsePositive); compareResults.Statistics.Intent.FalseNegative.Should().Be(falseNegative); actual = actual ?? "null"; expected = expected ?? "null"; compareResults.Statistics.ByIntent[expected].TruePositive.Should().Be(truePositive); compareResults.Statistics.ByIntent[expected].FalseNegative.Should().Be(falseNegative); compareResults.Statistics.ByIntent[actual].FalsePositive.Should().Be(falsePositive); }
public static void UsesIndexAsUtteranceId() { var expectedUtterance = new LabeledUtterance(null, "Greeting", null); var actualUtterance = new LabeledUtterance(null, "Greeting", null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance, expectedUtterance }, new[] { actualUtterance, actualUtterance }); compareResults.TestCases.Count.Should().Be(4); compareResults.TestCases.Where(t => t.UtteranceId == "0").Count().Should().Be(2); compareResults.TestCases.Where(t => t.UtteranceId == "1").Count().Should().Be(2); }
public static void GlobalLocalStrictIgnoreEntities( string expected, string actual, string value, bool unitTestMode, int truePositive, int trueNegative, int falsePositive, int falseNegative) { var globalSettingsFile = Guid.NewGuid().ToString(); var globalSettings = new JObject { { "strictEntities", new JArray { "c", "e" } }, { "ignoreEntities", new JArray { "d", "f" } }, }; File.WriteAllText(globalSettingsFile, globalSettings.ToString()); try { var expectedEntity = new Entity(expected, null, string.Empty, 0); var actualEntity = new Entity(actual, null, value, 0); var expectedUtterance = new LabeledUtterance(null, null, new[] { expectedEntity }) .WithProperty("strictEntities", new JArray { "a", "f" }) .WithProperty("ignoreEntities", new JArray { "b", "e" }); var actualUtterance = new LabeledUtterance(null, null, new[] { actualEntity }); var testSettings = new TestSettings(globalSettingsFile, unitTestMode); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, testSettings); compareResults.Statistics.Entity.TruePositive.Should().Be(truePositive); compareResults.Statistics.Entity.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Entity.FalsePositive.Should().Be(falsePositive); compareResults.Statistics.Entity.FalseNegative.Should().Be(falseNegative); } finally { File.Delete(globalSettingsFile); } }
public static void UsesInputUtteranceId() { var utteranceId = Guid.NewGuid().ToString(); var expectedUtterance = new CompareLabeledUtterance(utteranceId, null, "Greeting", null); var actualUtterance = new LabeledUtterance(null, "Greeting", null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, false); compareResults.TestCases.Count.Should().Be(2); compareResults.TestCases.Where(t => t.UtteranceId == utteranceId).Count().Should().Be(2); }
public static void UsesInputUtteranceId() { var utteranceId = Guid.NewGuid().ToString(); var expectedUtterance = new JsonLabeledUtterance(null, "Greeting", null); expectedUtterance.AdditionalProperties.Add("utteranceId", utteranceId); var actualUtterance = new LabeledUtterance(null, "Greeting", null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }); compareResults.TestCases.Count.Should().Be(2); compareResults.TestCases.Where(t => t.UtteranceId == utteranceId).Count().Should().Be(2); }
public static void GetNLUCompareResultsNullScores() { var entityType = Guid.NewGuid().ToString(); var matchText = Guid.NewGuid().ToString(); var expectedEntity = new[] { new Entity(entityType, null, matchText, 0) }; var actualEntity = new[] { new Entity(entityType, null, matchText, 0) }; var expectedUtterance = new LabeledUtterance(null, null, expectedEntity); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }); compareResults.TestCases.Select(t => t.Score).Any(score => score != null).Should().BeFalse(); }
public static void GetNLUCompareResultsExtractsFalsePositiveEntityScore() { var entityType = Guid.NewGuid().ToString(); var matchText = Guid.NewGuid().ToString(); var actualEntity = new[] { new Entity(entityType, null, matchText, 0).WithScore(0.5) }; var expectedUtterance = new LabeledUtterance(null, null, null); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }); var testCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Entity); testCase.Should().NotBeNull(); testCase.Score.Should().Be(0.5); }
public static void GetNLUCompareResultsDefaultScores() { var entityType = Guid.NewGuid().ToString(); var matchText = Guid.NewGuid().ToString(); var expectedEntity = new[] { new Entity(entityType, null, matchText, 0) }; var actualEntity = new[] { new Entity(entityType, null, matchText, 0) }; var expectedUtterance = new LabeledUtterance(null, null, expectedEntity); var actualUtterance = new LabeledUtterance(null, null, actualEntity); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, false); compareResults.TestCases.Select(t => t.Score).Should().AllBeEquivalentTo(0); }
public static void NoFalsePositiveIntentsUnitTestMode() { var expectedUtterance = new JsonLabeledUtterance(new JsonEntities(Array.Empty <Entity>())); var actualUtterance = new LabeledUtterance(null, "foo", null); var testSettings = new TestSettings(default(string), true); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, testSettings); compareResults.Statistics.Intent.TruePositive.Should().Be(0); compareResults.Statistics.Intent.TrueNegative.Should().Be(0); compareResults.Statistics.Intent.FalsePositive.Should().Be(0); compareResults.Statistics.Intent.FalseNegative.Should().Be(0); }
public static void GetNLUCompareResultsExtractsIntentAndTextScore() { var expectedUtterance = new LabeledUtterance(null, null, null).WithProperty("speechFile", "speechFile"); var actualUtterance = new LabeledUtterance(null, null, null).WithScore(0.5).WithTextScore(0.1); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }); var intentTestCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Intent); intentTestCase.Should().NotBeNull(); intentTestCase.Score.Should().Be(0.5); var textTestCase = compareResults.TestCases.FirstOrDefault(t => t.TargetKind == ComparisonTargetKind.Text); textTestCase.Should().NotBeNull(); textTestCase.Score.Should().Be(0.1); }
public static void GetNLUCompareResultsTextStatistics( string expected, string actual, int truePositive, int trueNegative, int falsePositive, int falseNegative) { var expectedUtterance = new LabeledUtterance(expected, null, null).WithProperty("speechFile", "speechFile"); var actualUtterance = new LabeledUtterance(actual, null, null); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }); compareResults.Statistics.Text.TruePositive.Should().Be(truePositive); compareResults.Statistics.Text.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Text.FalsePositive.Should().Be(falsePositive); compareResults.Statistics.Text.FalseNegative.Should().Be(falseNegative); }
public static void CustomTrueNegativeIntent( string expected, string actual, int truePositive, int trueNegative, int falsePositive, int falseNegative) { var expectedUtterance = new LabeledUtterance(null, expected, null); var actualUtterance = new LabeledUtterance(null, actual, null); var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary <string, string> { { "trueNegativeIntent", "foo" }, }) .Build(); var testSettings = new TestSettings(configuration, false); var compareResults = TestCaseSource.GetNLUCompareResults( new[] { expectedUtterance }, new[] { actualUtterance }, testSettings); compareResults.Statistics.Intent.TruePositive.Should().Be(truePositive); compareResults.Statistics.Intent.TrueNegative.Should().Be(trueNegative); compareResults.Statistics.Intent.FalsePositive.Should().Be(falsePositive); compareResults.Statistics.Intent.FalseNegative.Should().Be(falseNegative); if (expected != null && expected != "foo") { compareResults.Statistics.ByIntent[expected].TruePositive.Should().Be(truePositive); compareResults.Statistics.ByIntent[expected].FalseNegative.Should().Be(falseNegative); } else if (actual != null && actual != "foo") { compareResults.Statistics.ByIntent[actual].FalsePositive.Should().Be(falsePositive); } }