private static TestCase CreateTestCase(
string utteranceId,
ConfusionMatrixResultKind resultKind,
ComparisonTargetKind targetKind,
ILabeledUtterance expectedUtterance,
ILabeledUtterance actualUtterance,
double?score,
string group,
string[] args,
string because,
IEnumerable <string> categories)
{
var categoriesWithGroup = categories;
if (group != null)
{
categoriesWithGroup.Append(group);
}
return(new TestCase(
utteranceId,
resultKind,
targetKind,
expectedUtterance,
actualUtterance,
score,
group,
$"{resultKind}{targetKind}('{string.Join("', '", args)}')",
because,
categoriesWithGroup));
}
/// <summary>
/// Checks if a property is present in the JSON object.
/// </summary>
/// <param name="labeledUtterance">Labeled utterance.</param>
/// <param name="propertyName">Property name.</param>
/// <returns>
/// <code>true</code> if the property is present, otherwise <code>false</code>.
/// </returns>
public static bool HasProperty(this ILabeledUtterance labeledUtterance, string propertyName)
{
if (labeledUtterance is IJsonExtension jsonExtension)
{
return(jsonExtension.AdditionalProperties.TryGetValue(propertyName, out var unused));
}
throw new InvalidOperationException("Property existence cannot be checked.");
}
public LabeledUtteranceTestInput(
string utteranceId,
ILabeledUtterance expected,
ILabeledUtterance actual,
TestSettings testSettings)
{
this.UtteranceId = utteranceId;
this.Expected = expected;
this.Actual = actual;
this.TestSettings = testSettings;
}
private static string CreateSampleUtterance(ILabeledUtterance utterance)
{
var text = utterance.Text;
if (utterance.Entities != null)
{
foreach (var entity in utterance.Entities)
{
// Match in original text
var index = entity.StartCharIndexInText(utterance.Text);
// Replace the matching token with the slot indicator
text = new Regex(entity.MatchText)
.Replace(text, $"{{{entity.EntityType}}}", 1, index);
}
}
return(text);
}
public static JSONUtterance ToJSONUtterance(this ILabeledUtterance utterance, LuisApp luisApp)
{
JSONEntity toJSONEntity(IEntity entity)
{
var startPos = entity.StartCharIndexInText(utterance.Text);
var endPos = startPos + entity.MatchText.Length - 1;
if (luisApp.TryGetEntityTypeForRole(entity.EntityType, out var entityType))
{
return(new JSONEntityWithRole(startPos, endPos, entityType, entity.EntityType));
}
return(new JSONEntity(startPos, endPos, entity.EntityType));
}
return(new JSONUtterance(
utterance.Text,
utterance.Intent,
utterance.Entities?.Select(toJSONEntity).ToArray() ?? Array.Empty <JSONEntity>()));
}
private static TestCase FalseNegative(
string utteranceId,
ComparisonTargetKind targetKind,
ILabeledUtterance expectedUtterance,
ILabeledUtterance actualUtterance,
double?score,
string group,
string[] args,
string because,
params string[] categories)
{
return(CreateTestCase(
utteranceId,
ConfusionMatrixResultKind.FalseNegative,
targetKind,
expectedUtterance,
actualUtterance,
score,
group,
args,
because,
categories.Append("FalseNegative")));
}
/// <summary>
/// Initializes a new instance of the <see cref="TestCase"/> class.
/// </summary>
/// <param name="utteranceId">Utterance ID.</param>
/// <param name="resultKind">Confusion matrix result kind.</param>
/// <param name="targetKind">Comparison target kind.</param>
/// <param name="expectedUtterance">Expected utterance.</param>
/// <param name="actualUtterance">Actual utterance.</param>
/// <param name="score">Confidence score for test case result.</param>
/// <param name="group">Test case group name.</param>
/// <param name="testName">Test name.</param>
/// <param name="because">Because.</param>
/// <param name="categories">Categories.</param>
public TestCase(
string utteranceId,
ConfusionMatrixResultKind resultKind,
ComparisonTargetKind targetKind,
ILabeledUtterance expectedUtterance,
ILabeledUtterance actualUtterance,
double?score,
string group,
string testName,
string because,
IEnumerable <string> categories)
{
this.UtteranceId = utteranceId;
this.ResultKind = resultKind;
this.TargetKind = targetKind;
this.ExpectedUtterance = expectedUtterance;
this.ActualUtterance = actualUtterance;
this.Score = score;
this.Group = group;
this.TestName = testName;
this.Because = because;
this.Categories = categories.ToList();
}
private static bool IsStrictEntity(
string entityType,
ILabeledUtterance expectedUtterance,
TestSettings testSettings)
{
var localIgnoreEntities = expectedUtterance.GetIgnoreEntities();
var localStrictEntities = expectedUtterance.GetStrictEntities();
if (!testSettings.UnitTestMode)
{
var globalIgnoreEntities = testSettings.IgnoreEntities;
return(!localIgnoreEntities
.Union(globalIgnoreEntities)
.Except(localStrictEntities)
.Contains(entityType));
}
var globalStrictEntities = testSettings.StrictEntities;
return(localStrictEntities
.Union(globalStrictEntities)
.Except(localIgnoreEntities)
.Contains(entityType));
}
/// <summary>
/// Gets the timestamp for the labeled utterance.
/// </summary>
/// <param name="instance">Labeled utterance instance.</param>
/// <returns>
/// Timestamp, or <code>null</code> if property is not set.
/// </returns>
public static DateTimeOffset?GetTimestamp(this ILabeledUtterance instance)
{
return(instance.GetPropertyCore <DateTimeOffset?>(TimestampPropertyName));
}
private static ILabeledUtterance ToJsonExtension(this ILabeledUtterance utterance)
{
return(utterance is IJsonExtension
? utterance
: new LabeledUtterance(utterance.Text, utterance.Intent, utterance.Entities));
}
public TestCommandWithMockResult(ILabeledUtterance testResult, TestOptions options)
: base(options)
{
this.TestResult = testResult;
}
/// <summary>
/// Adds a confidence score for the intent label to the labeled utterance.
/// </summary>
/// <param name="instance">Labeled utterance instance.</param>
/// <param name="score">Confidence score.</param>
/// <returns>Labeled utterance with intent confidence score.</returns>
public static ILabeledUtterance WithScore(this ILabeledUtterance instance, double?score)
{
return(instance.WithProperty(ScorePropertyName, score));
}
public static IReadOnlyList <string> GetStrictEntities(this ILabeledUtterance utterance)
{
return(utterance.GetProperty <JArray>(StrictEntitiesPropertyName)?.ToObject <string[]>() ?? Array.Empty <string>());
}
public static string GetUtteranceId(this ILabeledUtterance utterance)
{
return(utterance.GetProperty <string>(UtteranceIdPropertyName));
}
/// <summary>
/// Gets the property for the labeled utterance.
/// </summary>
/// <typeparam name="T">Property value type.</typeparam>
/// <param name="instance">Labeled utterance instance.</param>
/// <param name="propertyName">Property name.</param>
/// <returns>
/// Property value, or default if property is not set.
/// </returns>
public static T GetProperty <T>(this ILabeledUtterance instance, string propertyName)
{
return(instance.GetPropertyCore <T>(propertyName));
}
/// <summary>
/// Gets the text transcription confidence score for the labeled utterance.
/// </summary>
/// <param name="instance">Labeled utterance instance.</param>
/// <returns>
/// Transcription confidence score, or <code>null</code> if property is not set.
/// </returns>
public static double?GetTextScore(this ILabeledUtterance instance)
{
return(instance.GetNumericProperty(TextScorePropertyName));
}
/// <summary>
/// Adds a property to the labeled utterance.
/// </summary>
/// <param name="instance">Labeled utterance instance.</param>
/// <param name="propertyName">Property name.</param>
/// <param name="propertyValue">Property value.</param>
/// <returns>Labeled utterance with additional property.</returns>
public static ILabeledUtterance WithProperty(this ILabeledUtterance instance, string propertyName, object propertyValue)
{
return(instance.WithProperty(propertyName, propertyValue, ToJsonExtension));
}
/// <summary>
/// Adds a timestamp to the labeled utterance.
/// </summary>
/// <param name="instance">Labeled utterance instance.</param>
/// <param name="timestamp">Timestamp.</param>
/// <returns>Labeled utterance with timestamp.</returns>
public static ILabeledUtterance WithTimestamp(this ILabeledUtterance instance, DateTimeOffset?timestamp)
{
return(instance.WithProperty(TimestampPropertyName, timestamp));
}
/// <summary>
/// Adds a confidence score for the text transcription to the labeled utterance.
/// </summary>
/// <param name="instance">Labeled utterance instance.</param>
/// <param name="textScore">Confidence score.</param>
/// <returns>Labeled utterance with transcription confidence score.</returns>
public static ILabeledUtterance WithTextScore(this ILabeledUtterance instance, double?textScore)
{
return(instance.WithProperty(TextScorePropertyName, textScore));
}
