public static void TestToEntityTestCasesMissingEntityInExpectedFile()
{
var actualEntity = CreateEntityList("EntityType");
actualEntity.Add(new Entity("RecognizedEntity", "value", "text", 2));
var expectedEntity = CreateEntityList("EntityType");
var utterances = CreatePair(new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(3);
var actualFalsePositive = testResult.FirstOrDefault(t => t.ResultKind == ConfusionMatrixResultKind.FalsePositive);
var actualTruePositive = testResult.FirstOrDefault(t => t.ResultKind == ConfusionMatrixResultKind.TruePositive);
var actualTruePositiveEntityValue = testResult.FirstOrDefault(t => Matches(t.TestName, TruePositiveEntityValueRegex));
actualFalsePositive.Should().NotBeNull();
actualFalsePositive.TestName.Should().MatchRegex(FalsePositiveEntityRegex);
actualTruePositive.Should().NotBeNull();
actualTruePositive.TestName.Should().MatchRegex(TruePositiveEntityRegex);
actualTruePositiveEntityValue.Should().NotBeNull();
}
public static void TestToEntityTestCasesWrongEntityType()
{
var actualEntity = CreateEntityList("EntityType");
var expectedEntity = CreateEntityList("WrongType");
var utterances = new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
};
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(3);
var actualFalsePositive = testResult.FirstOrDefault(t => t.Kind == TestResultKind.FalsePositive);
var actualFalseNegative = testResult.FirstOrDefault(t => t.Kind == TestResultKind.FalseNegative);
var actualFalseNegativeEntityValue = testResult.FirstOrDefault(t => Matches(t.Message, FalseNegativeEntityValueRegex));
actualFalsePositive.Should().NotBeNull();
actualFalsePositive.TestName.Should().MatchRegex(FalsePositiveEntityRegex);
actualFalseNegative.Should().NotBeNull();
actualFalseNegative.TestName.Should().MatchRegex(FalseNegativeEntityRegex);
actualFalseNegativeEntityValue.Should().NotBeNull();
}
public static void TestToEntityTestCasesMissingAnActualEntity()
{
var entity = CreateEntityList("EntityType");
var utterances = CreatePair(new[]
{
new LabeledUtterance("FOO", "DayTime", entity),
new LabeledUtterance("FOO", "DayTime", null)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
actualTestResult.First().ResultKind.Should().Be(ConfusionMatrixResultKind.FalseNegative);
actualTestResult.First().TestName.Should().MatchRegex(FalseNegativeEntityRegex);
}
public static void ToEntityTestCasesWithNullActualMatchText()
{
var actualEntity = new[] { new Entity("EntityType", "foo", null, 0) };
var expectedEntity = new[] { new Entity("EntityType", null, "foo", 0) };
var utterances = CreatePair(new[]
{
new LabeledUtterance(null, null, expectedEntity),
new LabeledUtterance(null, null, actualEntity)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(1);
testResult.Should().Contain(t => Matches(t.TestName, TruePositiveEntityRegex));
}
public static void ToEntityTestCasesWithIncorrectMatchIndex()
{
var actualEntity = new[] { new Entity("EntityType", "EntityValue", "differentMatchedText", 1) };
var expectedEntity = new[] { new Entity("EntityType", null, "differentMatchedText", 2) };
var utterances = CreatePair(new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(2);
testResult.Should().Contain(t => Matches(t.TestName, FalsePositiveEntityRegex));
testResult.Should().Contain(t => Matches(t.TestName, FalseNegativeEntityRegex));
}
public static void ToEntityTestCasesWithFalseNegativeEntityValueMismatch()
{
var actualEntity = new[] { new Entity("EntityType", "qux", "foo", 0) };
var expectedEntity = new[] { new Entity("EntityType", "bar", "foo", 0) };
var utterances = new[]
{
new LabeledUtterance(null, null, expectedEntity),
new LabeledUtterance(null, null, actualEntity)
};
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(2);
testResult.Should().Contain(t => Matches(t.TestName, TruePositiveEntityRegex));
testResult.Should().Contain(t => Matches(t.TestName, FalseNegativeEntityValueRegex));
}
public static void ToEntityTestCasesWithNullExpectedEntityValue()
{
foreach (var expectedEntityValue in new[] { JValue.CreateNull(), null })
{
var actualEntity = new[] { new Entity("EntityType", 42, "differentMatchedText", 1) };
var expectedEntity = new[] { new Entity("EntityType", expectedEntityValue, "differentMatchedText", 1) };
var utterances = CreatePair(new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(1);
testResult.Should().NotContain(t => Matches(t.TestName, TruePositiveEntityValueRegex));
}
}
public static void TestToEntityTestCasesMissingEntityInActualResult()
{
var expectedEntity = CreateEntityList("EntityType");
var utterances = CreatePair(new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", null)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(1);
var actualFalseNegative = testResult.FirstOrDefault(t => t.ResultKind == ConfusionMatrixResultKind.FalseNegative);
actualFalseNegative.Should().NotBeNull();
actualFalseNegative.TestName.Should().MatchRegex(FalseNegativeEntityRegex);
}
public static void ToEntityTestCasesWithNullExpectedEntityResolution()
{
var actualEntity = new List <Entity> {
new Entity("EntityType", "EntityValue", "foo", "differentMatchedText", 1)
};
var expectedEntity = new List <Entity> {
new Entity("EntityType", "EntityValue", JValue.CreateNull(), "differentMatchedText", 1)
};
var utterances = new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
};
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(2);
testResult.Should().NotContain(t => Matches(t.Message, TruePositiveEntityResolutionRegex));
}
public static void ToEntityTestCasesWithEntityValue(string expectedJson, string actualJson, string testMessageRegex)
{
var actualEntity = new List <Entity> {
new Entity("EntityType", ParseEntityValueJson(actualJson), "foo", 1)
};
var expectedEntity = new List <Entity> {
new Entity("EntityType", ParseEntityValueJson(expectedJson), "foo", 1)
};
var utterances = CreatePair(new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(2);
var actual = testResult.FirstOrDefault(t => Matches(t.TestName, testMessageRegex));
actual.Should().NotBeNull();
}
public static void TestToEntityTestCasesWithDifferentEntityValueInActualVersusExpected()
{
var actualEntity = new[] { new Entity("EntityType", "differentEntityValue", "differentMatchedText", 1) };
var expectedEntity = CreateEntityList("EntityType");
var utterances = CreatePair(new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
});
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(2);
var actualFalsePositive = testResult.FirstOrDefault(t => t.ResultKind == ConfusionMatrixResultKind.FalsePositive);
var actualFalseNegative = testResult.FirstOrDefault(t => t.ResultKind == ConfusionMatrixResultKind.FalseNegative);
actualFalsePositive.Should().NotBeNull();
actualFalsePositive.TestName.Should().MatchRegex(FalsePositiveEntityRegex);
actualFalseNegative.Should().NotBeNull();
actualFalseNegative.TestName.Should().MatchRegex(FalseNegativeEntityRegex);
}
public static void ToEntityTestCasesWithEntityResolution(string expectedJson, string actualJson, string testMessageRegex)
{
var actualEntity = new List <Entity> {
new Entity("EntityType", "EntityValue", ParseResolutionJson(actualJson), "differentMatchedText", 1)
};
var expectedEntity = new List <Entity> {
new Entity("EntityType", "EntityValue", ParseResolutionJson(expectedJson), "differentMatchedText", 1)
};
var utterances = new[]
{
new LabeledUtterance("FOO", "DayTime", expectedEntity),
new LabeledUtterance("FOO", "DayTime", actualEntity)
};
var actualTestResult = TestCaseSource.ToEntityTestCases(utterances);
var testResult = actualTestResult.ToList();
testResult.Count().Should().Be(3);
var actual = testResult.FirstOrDefault(t => Matches(t.Message, testMessageRegex));
actual.Should().NotBeNull();
}