private static async Task ExpectDownloadModelAsync(IBlobContainer modelContainer, Guid modelId)
{
await
modelContainer.Received(1)
.DownloadBlobAsync(ModelsProvider.GetModelBlobName(modelId), Arg.Any <Stream>(),
Arg.Any <CancellationToken>());
}
/// <summary>
/// Creates anew instance of the <see cref="WebJobLogic"/> class.
/// </summary>
/// <param name="modelsProvider">A models provider</param>
/// <param name="modelsRegistry">A model registry</param>
public WebJobLogic(ModelsProvider modelsProvider, ModelsRegistry modelsRegistry)
{
if (modelsProvider == null)
{
throw new ArgumentNullException(nameof(modelsProvider));
}
if (modelsRegistry == null)
{
throw new ArgumentNullException(nameof(modelsRegistry));
}
_modelsProvider = modelsProvider;
_modelsRegistry = modelsRegistry;
}
public async Task ScoreMultipleModelsAsyncTest()
{
var modelsContainer = CreateModelContainer();
var containerProvider = Substitute.For <IBlobContainerProvider>();
var modelId1 = Guid.NewGuid();
var modelId2 = Guid.NewGuid();
var modelId3 = Guid.NewGuid();
var usageEvents = new[] { new UsageEvent {
ItemId = "item.0"
} };
var trainedModelSource = Path.Combine(nameof(ScoreMultipleModelsAsyncTest), "model.zip");
ProvideModel(modelsContainer, modelId1, trainedModelSource);
ProvideModel(modelsContainer, modelId2, trainedModelSource);
ProvideModel(modelsContainer, modelId3, trainedModelSource);
containerProvider.GetBlobContainer(ModelsProvider.ModelsBlobContainerName, Arg.Any <bool>()).Returns(modelsContainer);
// create model provider with substitutes
var modelProvider = CreateModelProvider(containerProvider);
// get recommendations
var scoreResult = await modelProvider.ScoreAsync(modelId1, usageEvents, null, 5, CancellationToken.None);
Assert.AreEqual(5, scoreResult.Count, "did not receive the expected number of scoring results");
scoreResult = await modelProvider.ScoreAsync(modelId2, usageEvents, null, 1, CancellationToken.None);
Assert.AreEqual(1, scoreResult.Count, "did not receive the expected number of scoring results");
scoreResult = await modelProvider.ScoreAsync(modelId1, usageEvents, null, 5, CancellationToken.None);
Assert.AreEqual(5, scoreResult.Count, "did not receive the expected number of scoring results");
scoreResult = await modelProvider.ScoreAsync(modelId2, usageEvents, null, 10, CancellationToken.None);
Assert.AreEqual(10, scoreResult.Count, "did not receive the expected number of scoring results");
// expect call to provider substitute
await ExpectDownloadModelAsync(modelsContainer, modelId1);
await ExpectDownloadModelAsync(modelsContainer, modelId2);
await
modelsContainer.DidNotReceive()
.DownloadBlobAsync(ModelsProvider.GetModelBlobName(modelId3), Arg.Any <Stream>(),
Arg.Any <CancellationToken>());
}
private static async Task ExpectDeleteModelAsync(IBlobContainer modelContainer, Guid modelId)
{
await
modelContainer.Received(1)
.DeleteBlobIfExistsAsync(ModelsProvider.GetModelBlobName(modelId), Arg.Any <CancellationToken>());
}
private static void ProvideModel(IBlobContainer blobContainer, Guid modelId, string modelSourceFilePath)
{
ProvideStream(blobContainer, ModelsProvider.GetModelBlobName(modelId), modelSourceFilePath);
}
public async Task TrainAndScoreAsyncTest()
{
var modelsContainer = CreateModelContainer();
var trainContainer = Substitute.For <IBlobContainer>();
var containerProvider = Substitute.For <IBlobContainerProvider>();
var catalogSource = Path.Combine(nameof(TrainAndScoreAsyncTest), "catalog.csv");
var usageSource = Path.Combine(nameof(TrainAndScoreAsyncTest), "usage");
var evaluationSource = Path.Combine(nameof(TrainAndScoreAsyncTest), "evaluation");
var trainContainerName = "trainContainer";
var modelId = Guid.NewGuid();
var trainedModelBlobName = ModelsProvider.GetModelBlobName(modelId);
using (var trainedModelStream = new MemoryStream())
{
AcceptBlob(modelsContainer, trainedModelBlobName, trainedModelStream);
ProvideTrainBlobContainer(trainContainer, usageSource, catalogSource, evaluationSource);
containerProvider.GetBlobContainer(ModelsProvider.ModelsBlobContainerName, Arg.Any <bool>()).Returns(modelsContainer);
containerProvider.GetBlobContainer(trainContainerName, Arg.Any <bool>()).Returns(trainContainer);
// create model provider with substitutes
var modelProvider = CreateModelProvider(containerProvider);
// train model
var trainingParameters = CreateModelTrainingParameters(trainContainerName, true, true);
var trainResult = await modelProvider.TrainAsync(modelId, trainingParameters, null, CancellationToken.None);
Assert.IsTrue(trainResult.IsCompletedSuccessfuly, "training failed did not complete successfully");
// expect the train blobs container to be listed
await trainContainer.Received(1).ListBlobsAsync(UsageFolderRelativeLocation, Arg.Any <CancellationToken>());
await trainContainer.Received(1).ListBlobsAsync(EvaluationFolderRelativeLocation, Arg.Any <CancellationToken>());
// expect every usage blob to be downloaded
await ExpectDownloadBlobAsync(trainContainer, CatalogFileRelativeLocation);
await ExpectDownloadAllBlobsAsync(trainContainer, UsageFolderRelativeLocation, usageSource);
await ExpectDownloadAllBlobsAsync(trainContainer, EvaluationFolderRelativeLocation, evaluationSource);
// expect a trained model to be uploaded to models container
await ExpectUploadModelAsync(modelsContainer, modelId);
// score with the trained model
// provide uploaded stream
trainedModelStream.Seek(0, SeekOrigin.Begin);
modelsContainer.DownloadBlobAsync(trainedModelBlobName, Arg.Any <Stream>(), Arg.Any <CancellationToken>())
.Returns(args => trainedModelStream.CopyToAsync(args[1] as Stream));
// get recommendations
var recommendationsCount = 5;
var scoreResult = await modelProvider.ScoreAsync(
modelId,
new[] { new UsageEvent {
ItemId = "item.0"
} },
null,
recommendationsCount, CancellationToken.None);
// expect the requested number of recommendations
Assert.AreEqual(recommendationsCount, scoreResult.Count, "did not receive the expected number of scoring results");
// expect call to provider substitute
await ExpectDownloadModelAsync(modelsContainer, modelId);
}
}
