public static async Task TrainEmptyModel()
{
var mockClient = new MockLuisClient();
var builder = GetTestLuisBuilder();
builder.AppVersion = Guid.NewGuid().ToString();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
var utterances = Array.Empty <Models.LabeledUtterance>();
await luis.TrainAsync(utterances).ConfigureAwait(false);
// Assert correct import request
var importRequest = mockClient.Requests.FirstOrDefault(request => request.Method == nameof(ILuisClient.ImportVersionAsync));
importRequest.Should().NotBeNull();
importRequest.Arguments[2].Should().NotBeNull();
var luisApp = importRequest.Arguments[2].As <LuisApp>();
// Expects 3 intents
luisApp.Intents.Count.Should().Be(1);
luisApp.Intents.First().Name.Should().Be("None");
// Assert train request
var trainRequest = mockClient.Requests.FirstOrDefault(request => request.Method == nameof(ILuisClient.TrainAsync));
trainRequest.Should().NotBeNull();
// Assert publish request
var publishRequest = mockClient.Requests.FirstOrDefault(request => request.Method == nameof(ILuisClient.PublishAppAsync));
publishRequest.Should().NotBeNull();
// Expects publish settings:
publishRequest.Arguments[0].Should().Be(builder.AppId);
publishRequest.Arguments[1].Should().Be(builder.AppVersion);
}
}
public static async Task TagsPrebuiltEntityWithReplacementName()
{
var text = Guid.NewGuid().ToString();
var entityTypeName1 = Guid.NewGuid().ToString();
var entityTypeName2 = Guid.NewGuid().ToString();
var prebuiltEntityTypes = new Dictionary <string, string>
{
{ entityTypeName1, entityTypeName2 },
};
var mockClient = new MockLuisClient();
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
builder.LuisSettings = new LuisSettings(prebuiltEntityTypes);
using (var luis = builder.Build())
{
var entity1 = new Entity(entityTypeName1, null, text, 0);
var entity2 = new Entity(entityTypeName2, null, text, 0);
var utterance = new Models.LabeledUtterance(text, string.Empty, new[] { entity1, entity2 });
await luis.TrainAsync(new[] { utterance }).ConfigureAwait(false);
// Ensure LUIS app intent still has role
var importRequest = mockClient.Requests.FirstOrDefault(request => request.Method == nameof(ILuisClient.ImportVersionAsync));
importRequest.Should().NotBeNull();
var luisApp = importRequest.Arguments[2].As <LuisApp>();
luisApp.Utterances.Should().Contain(u => u.Text == text);
luisApp.Utterances.First(u => u.Text == text).Entities.Count().Should().Be(2);
luisApp.Utterances.First(u => u.Text == text).Entities.Should().Contain(e => e.Entity == entityTypeName2);
luisApp.Utterances.First(u => u.Text == text).Entities.Should().Contain(e => e.Entity == entityTypeName2);
}
}
public static async Task WithLabeledIntentScore()
{
var test = "the quick brown fox jumped over the lazy dog";
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (request.Method == nameof(ILuisClient.QueryAsync))
{
return(new LuisResult
{
Query = test,
TopScoringIntent = new IntentModel {
Intent = "intent", Score = 0.42
},
});
}
return(null);
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
var result = await luis.TestAsync(test).ConfigureAwait(false);
result.Should().BeOfType(typeof(ScoredLabeledUtterance));
result.As <ScoredLabeledUtterance>().Score.Should().Be(0.42);
}
}
public static async Task CreatesAppIfAppIdNotProvided()
{
var appId = Guid.NewGuid().ToString();
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (request.Method == nameof(ILuisClient.CreateAppAsync))
{
return(appId);
}
return(null);
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
builder.AppId = null;
using (var luis = builder.Build())
{
await luis.TrainAsync(Array.Empty <Models.LabeledUtterance>()).ConfigureAwait(false);
luis.LuisAppId.Should().Be(appId);
}
}
public static void TrainingFailedThrowsInvalidOperation()
{
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (IsTrainingStatusRequest(request))
{
return(new[]
{
new ModelTrainingInfo
{
Details = new ModelTrainingDetails {
Status = "Fail"
}
}
});
}
return(null);
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
Func <Task> trainAsync = () => luis.TrainAsync(Array.Empty <Models.LabeledUtterance>());
trainAsync.Should().Throw <InvalidOperationException>();
}
}
public static async Task TestWithPrebuiltEntity()
{
var test = "the quick brown fox jumped over the lazy dog";
var builtinType = Guid.NewGuid().ToString();
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (request.Method == nameof(ILuisClient.QueryAsync))
{
return(new LuisResult
{
Query = test,
TopScoringIntent = new IntentModel {
Intent = "intent"
},
Entities = new[]
{
new EntityModel
{
Entity = "the",
Type = "builtin.test",
StartIndex = 32,
EndIndex = 34
},
},
});
}
return(null);
};
var prebuiltEntityTypes = new Dictionary <string, string>
{
{ "type", "test" },
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
builder.LuisSettings = new LuisSettings(prebuiltEntityTypes);
using (var luis = builder.Build())
{
var result = await luis.TestAsync(test).ConfigureAwait(false);
result.Text.Should().Be(test);
result.Intent.Should().Be("intent");
result.Entities.Count.Should().Be(1);
result.Entities[0].EntityType.Should().Be("type");
result.Entities[0].EntityValue.Should().Be(default(string));
result.Entities[0].MatchText.Should().Be("the");
result.Entities[0].MatchIndex.Should().Be(1);
}
}
public static async Task TrainModelWithUtterancesAndSimpleEntities()
{
var mockClient = new MockLuisClient();
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
var utterances = new[]
{
new Models.LabeledUtterance(
"Book me a flight.",
"BookFlight",
new Entity[] { new Entity("Name", string.Empty, "me", 0) }),
new Models.LabeledUtterance(
"Cancel my flight.",
"CancelFlight",
new Entity[] { new Entity("Subject", string.Empty, "flight", 0) })
};
await luis.TrainAsync(utterances).ConfigureAwait(false);
// Assert correct import request
var importRequest = mockClient.Requests.FirstOrDefault(request => request.Method == nameof(ILuisClient.ImportVersionAsync));
importRequest.Should().NotBeNull();
var luisApp = importRequest.Arguments[2].As <LuisApp>();
// Expects 3 intents
luisApp.Intents.Count.Should().Be(3);
luisApp.Intents.Should().Contain(intent => intent.Name == "None");
luisApp.Intents.Should().Contain(intent => intent.Name == utterances[0].Intent);
luisApp.Intents.Should().Contain(intent => intent.Name == utterances[1].Intent);
// Expect 2 utterances
luisApp.Utterances.Count.Should().Be(2);
var bookUtterance = luisApp.Utterances.FirstOrDefault(utterance => utterance.Intent == utterances[0].Intent);
bookUtterance.Should().NotBeNull();
bookUtterance.Text.Should().Be(utterances[0].Text);
bookUtterance.Entities.Count.Should().Be(1);
bookUtterance.Entities.First().Entity.Should().Be(utterances[0].Entities[0].EntityType);
bookUtterance.Entities.First().StartPos.Should().Be(5);
bookUtterance.Entities.First().EndPos.Should().Be(6);
var cancelUtterance = luisApp.Utterances.FirstOrDefault(utterance => utterance.Intent == utterances[1].Intent);
cancelUtterance.Should().NotBeNull();
cancelUtterance.Text.Should().Be(utterances[1].Text);
cancelUtterance.Entities.Count.Should().Be(1);
cancelUtterance.Entities.First().Entity.Should().Be(utterances[1].Entities[0].EntityType);
cancelUtterance.Entities.First().StartPos.Should().Be(10);
cancelUtterance.Entities.First().EndPos.Should().Be(15);
}
}
public static async Task CleanupModel()
{
var mockClient = new MockLuisClient();
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
await luis.CleanupAsync().ConfigureAwait(false);
var cleanupRequest = mockClient.Requests.FirstOrDefault(request => request.Method == nameof(ILuisClient.DeleteAppAsync));
cleanupRequest.Should().NotBeNull();
}
}
public static async Task WithEntityScore()
{
var test = "the quick brown fox jumped over the lazy dog";
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (request.Method == nameof(ILuisClient.QueryAsync))
{
return(new LuisResult
{
Query = test,
TopScoringIntent = new IntentModel {
Intent = "intent"
},
Entities = new[]
{
new EntityModel
{
Entity = "the",
Type = "type",
StartIndex = 32,
EndIndex = 34,
AdditionalProperties = new Dictionary <string, object>
{
{ "score", 0.42 },
},
},
},
});
}
return(null);
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
var result = await luis.TestAsync(test).ConfigureAwait(false);
result.Entities.Count.Should().Be(1);
result.Entities[0].Should().BeOfType(typeof(ScoredEntity));
result.Entities[0].As <ScoredEntity>().Score.Should().Be(0.42);
}
}
public static async Task TestSpeech()
{
var test = "the quick brown fox jumped over the lazy dog entity";
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (request.Method == nameof(ILuisClient.RecognizeSpeechAsync))
{
return(new LuisResult
{
Query = test,
TopScoringIntent = new IntentModel {
Intent = "intent"
},
Entities = new[]
{
new EntityModel
{
Entity = "entity",
Type = "type",
StartIndex = 45,
EndIndex = 50,
},
},
});
}
return(null);
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
var result = await luis.TestSpeechAsync("somefile").ConfigureAwait(false);
result.Text.Should().Be(test);
result.Intent.Should().Be("intent");
result.Entities.Count.Should().Be(1);
result.Entities[0].EntityType.Should().Be("type");
result.Entities[0].MatchText.Should().Be("entity");
result.Entities[0].MatchIndex.Should().Be(0);
}
}
public static async Task TrainingStatusDelayBetweenPolling()
{
var count = 0;
string[] statusArray = { "Queued", "InProgress", "Success" };
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (IsTrainingStatusRequest(request))
{
return(new[]
{
new ModelTrainingInfo
{
Details = new ModelTrainingDetails {
Status = statusArray[count++]
}
}
});
}
return(null);
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
await luis.TrainAsync(Array.Empty <Models.LabeledUtterance>()).ConfigureAwait(false);
// Ensure correct number of training status requests are made.
mockClient.Requests.Where(IsTrainingStatusRequest).Count().Should().Be(statusArray.Length);
// Ensure 2 second delay between requests
var previousRequest = mockClient.TimestampedRequests.Where(t => IsTrainingStatusRequest(t.Instance)).First();
for (var i = 1; i < statusArray.Length; ++i)
{
var nextRequest = mockClient.TimestampedRequests.Where(t => IsTrainingStatusRequest(t.Instance)).Skip(i).First();
var timeDifference = nextRequest.Timestamp - previousRequest.Timestamp;
previousRequest = nextRequest;
timeDifference.Should().BeGreaterThan(TimeSpan.FromSeconds(2) - Epsilon);
}
}
}
public static async Task DoesNotOverwriteTemplateIntents()
{
var role = Guid.NewGuid().ToString();
var intentName = Guid.NewGuid().ToString();
var appTemplate = new LuisApp
{
ClosedLists = new List <ClosedList>(),
Entities = new List <HierarchicalModel>(),
Intents = new List <HierarchicalModel>
{
new HierarchicalModel
{
Name = intentName,
Roles = new List <string> {
role
},
},
},
ModelFeatures = new List <JSONModelFeature>(),
PrebuiltEntities = new List <PrebuiltEntity>(),
};
var mockClient = new MockLuisClient();
var builder = GetTestLuisBuilder();
builder.LuisSettings = new LuisSettings(appTemplate);
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
var utterance = new Models.LabeledUtterance(null, intentName, null);
await luis.TrainAsync(new[] { utterance }).ConfigureAwait(false);
// Ensure LUIS app intent still has role
var importRequest = mockClient.Requests.FirstOrDefault(request => request.Method == nameof(ILuisClient.ImportVersionAsync));
importRequest.Should().NotBeNull();
var luisApp = importRequest.Arguments[2].As <LuisApp>();
luisApp.Intents.Should().Contain(intent => intent.Name == intentName);
luisApp.Intents.First(intent => intent.Name == intentName).Roles.Count.Should().Be(1);
luisApp.Intents.First(intent => intent.Name == intentName).Roles.First().Should().Be(role);
}
}
public static async Task TestModelWithEntityResolution()
{
var test = "the quick brown fox jumped over the lazy dog";
var mockClient = new MockLuisClient();
mockClient.OnRequestResponse = request =>
{
if (request.Method == nameof(ILuisClient.QueryAsync))
{
return(new LuisResult
{
Query = "the quick brown fox jumped over the lazy dog today",
Entities = new[]
{
new EntityModel
{
Entity = "today",
StartIndex = 45,
EndIndex = 49,
AdditionalProperties = new Dictionary <string, object>
{
{
"resolution",
new JObject
{
{ "values", new JArray {
new JObject {
{ "value", "2018-11-16" }
}
} },
}
},
},
},
new EntityModel
{
Entity = "brown fox",
StartIndex = 10,
EndIndex = 18,
AdditionalProperties = new Dictionary <string, object>
{
{ "resolution", new JObject {
{ "values", new JArray {
"Fox"
} }
} },
},
},
new EntityModel
{
Entity = "the",
StartIndex = 0,
EndIndex = 2,
AdditionalProperties = new Dictionary <string, object>
{
{ "resolution", new JObject {
{ "value", "THE" }
} },
},
}
}
});
}
return(null);
};
var builder = GetTestLuisBuilder();
builder.LuisClient = mockClient;
using (var luis = builder.Build())
{
var result = await luis.TestAsync(test).ConfigureAwait(false);
result.Entities.Count.Should().Be(3);
result.Entities[0].EntityValue.Should().Be("2018-11-16");
result.Entities[1].EntityValue.Should().Be("Fox");
result.Entities[2].EntityValue.Should().Be("THE");
}
}
