private static void ProvideBlob(IBlobContainer blobContainer, string name, string sourceFilePath)
{
blobContainer.DownloadBlobAsync(name, Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(args =>
{
File.Copy(sourceFilePath, args[1] as string);
return(Task.FromResult(0));
});
}
private static void ProvideStream(IBlobContainer blobContainer, string name, string sourceFilePath)
{
blobContainer.DownloadBlobAsync(name, Arg.Any <Stream>(), Arg.Any <CancellationToken>())
.Returns(args =>
{
using (var reader = new FileStream(sourceFilePath, FileMode.Open))
{
reader.CopyTo((Stream)args[1]);
}
return(Task.FromResult(0));
});
}
/// <summary>
/// Downloads the catalog, usage event files and evaluation usage files to local disk.
/// </summary>
private async Task <TrainingLocalFilePaths> DownloadTrainingBlobsAsync(Guid modelId, string localRootPath,
ModelTrainingParameters trainingParameters, CancellationToken cancellationToken)
{
try
{
var trainingFiles = new TrainingLocalFilePaths
{
// set local usage directory name
UsageFolderPath = Path.Combine(localRootPath, UsageDirectoryName)
};
// create the local folder for usage events files
Directory.CreateDirectory(trainingFiles.UsageFolderPath);
// get the root blob container of the catalog and usage files
IBlobContainer trainingBlobsContainer =
_blobContainerProvider.GetBlobContainer(trainingParameters.BlobContainerName);
// check if the provided path represents a single file
if (await trainingBlobsContainer.ExistsAsync(trainingParameters.UsageRelativePath, cancellationToken))
{
string usageEventsBlobName = trainingParameters.UsageRelativePath;
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath = Path.Combine(trainingFiles.UsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
else
{
Trace.TraceInformation(
$"Listing all the usage events blobs under '{trainingParameters.UsageRelativePath}'");
IList <string> usageEventsBlobNames = await trainingBlobsContainer.ListBlobsAsync(
trainingParameters.UsageRelativePath, cancellationToken);
Trace.TraceInformation(
$"Downloading all the usage events blobs (Found {usageEventsBlobNames.Count})");
foreach (string usageEventsBlobName in usageEventsBlobNames)
{
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath = Path.Combine(trainingFiles.UsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
}
// download the catalog file, if provided
if (!string.IsNullOrWhiteSpace(trainingParameters.CatalogFileRelativePath))
{
// set local catalog file path
var catalogFileName = Path.GetFileName(trainingParameters.CatalogFileRelativePath);
trainingFiles.CatalogFilePath = Path.Combine(localRootPath, catalogFileName);
Trace.TraceInformation($"Downloading catalog blob '{trainingFiles.CatalogFilePath}'");
await trainingBlobsContainer.DownloadBlobAsync(trainingParameters.CatalogFileRelativePath,
trainingFiles.CatalogFilePath, cancellationToken);
}
// download the evaluation files if provided
if (!string.IsNullOrWhiteSpace(trainingParameters.EvaluationUsageRelativePath))
{
// set local evaluation folder
trainingFiles.EvaluationUsageFolderPath = Path.Combine(localRootPath, EvaluationUsageLocalDirectoryName);
// create the local folder for evaluation usage events files
Directory.CreateDirectory(trainingFiles.EvaluationUsageFolderPath);
// check if the provided path represents a single file
if (await trainingBlobsContainer.ExistsAsync(trainingParameters.EvaluationUsageRelativePath,
cancellationToken))
{
string usageEventsBlobName = trainingParameters.EvaluationUsageRelativePath;
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath =
Path.Combine(trainingFiles.EvaluationUsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading evaluation usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
else
{
Trace.TraceInformation(
$"Listing all the evaluation usage events blobs under '{trainingParameters.EvaluationUsageRelativePath}'");
IList <string> evaluationUsageEventsBlobNames = await trainingBlobsContainer.ListBlobsAsync(
trainingParameters.EvaluationUsageRelativePath, cancellationToken);
Trace.TraceInformation(
$"Downloading all the evaluation usage events blobs (Found {evaluationUsageEventsBlobNames.Count})");
foreach (string usageEventsBlobName in evaluationUsageEventsBlobNames)
{
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath =
Path.Combine(trainingFiles.EvaluationUsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading evaluation usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
}
}
return(trainingFiles);
}
catch (StorageException storageException)
{
var exception = new Exception($"Failed downloading training files from storage. Model id: {modelId}",
storageException);
Trace.TraceError(exception.ToString());
throw exception;
}
}
