public async Task File_Should_Exists()
{
// Arrange
var bytes = new byte[] { 0x01, 0x02 };
// Act
await _blobContainer.SaveAsync("TestFile2", bytes);
// Assert
var isExists = await _blobContainer.ExistsAsync("TestFile2");
isExists.ShouldBe(true);
}
public Task <bool> ExistsAsync(
string name,
CancellationToken cancellationToken = default)
{
return(_container.ExistsAsync(
name,
cancellationToken
));
}
private static void ProvideBlob(IBlobContainer blobContainer, string name, string sourceFilePath)
{
blobContainer.ExistsAsync(name, Arg.Any <CancellationToken>()).Returns(Task.FromResult(true));
blobContainer.DownloadBlobAsync(name, Arg.Any <string>(), Arg.Any <CancellationToken>())
.Returns(args =>
{
File.Copy(sourceFilePath, args[1] as string);
return(Task.FromResult(0));
});
}
public async Task <DocumentMetadata> SaveDocumentAsync(Document document)
{
// save the content if it doesn't already exist
string documentHash = _hashAlgorithm.GetHash(document.Content);
bool duplicate = await _blobContainer.ExistsAsync(documentHash);
if (duplicate)
{
_eventSource.OnDuplicateFoundInDocumentStore(document.Id, documentHash);
}
else
{
// compress the document
byte[] compressedContent = _compressor.Compress(document.Content);
await _blobContainer.SetAsync(documentHash, compressedContent);
}
// save the metadata, in reverse chronological order
string documentMetadataId = string.Format(
CultureInfo.InvariantCulture,
"{0}-{1}",
DateTimeOffset.UtcNow.GetDescendingOrderString(),
Guid.NewGuid());
var documentMetadata = new DocumentMetadata
{
Id = documentMetadataId,
DocumentId = document.Id,
Hash = documentHash,
Duplicate = duplicate,
Created = DateTimeOffset.UtcNow
};
await _table.SetAsync(GetDocumentMetadataPartitionKey(document.Id), documentMetadataId, documentMetadata);
return(documentMetadata);
}
public virtual Task <bool> ExistsAsync(string path, CancellationToken cancellationToken = default)
{
return(_blobContainer.ExistsAsync(path, cancellationToken));
}
/// <summary>
/// Downloads the catalog, usage event files and evaluation usage files to local disk.
/// </summary>
private async Task <TrainingLocalFilePaths> DownloadTrainingBlobsAsync(Guid modelId, string localRootPath,
ModelTrainingParameters trainingParameters, CancellationToken cancellationToken)
{
try
{
var trainingFiles = new TrainingLocalFilePaths
{
// set local usage directory name
UsageFolderPath = Path.Combine(localRootPath, UsageDirectoryName)
};
// create the local folder for usage events files
Directory.CreateDirectory(trainingFiles.UsageFolderPath);
// get the root blob container of the catalog and usage files
IBlobContainer trainingBlobsContainer =
_blobContainerProvider.GetBlobContainer(trainingParameters.BlobContainerName);
// check if the provided path represents a single file
if (await trainingBlobsContainer.ExistsAsync(trainingParameters.UsageRelativePath, cancellationToken))
{
string usageEventsBlobName = trainingParameters.UsageRelativePath;
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath = Path.Combine(trainingFiles.UsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
else
{
Trace.TraceInformation(
$"Listing all the usage events blobs under '{trainingParameters.UsageRelativePath}'");
IList <string> usageEventsBlobNames = await trainingBlobsContainer.ListBlobsAsync(
trainingParameters.UsageRelativePath, cancellationToken);
Trace.TraceInformation(
$"Downloading all the usage events blobs (Found {usageEventsBlobNames.Count})");
foreach (string usageEventsBlobName in usageEventsBlobNames)
{
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath = Path.Combine(trainingFiles.UsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
}
// download the catalog file, if provided
if (!string.IsNullOrWhiteSpace(trainingParameters.CatalogFileRelativePath))
{
// set local catalog file path
var catalogFileName = Path.GetFileName(trainingParameters.CatalogFileRelativePath);
trainingFiles.CatalogFilePath = Path.Combine(localRootPath, catalogFileName);
Trace.TraceInformation($"Downloading catalog blob '{trainingFiles.CatalogFilePath}'");
await trainingBlobsContainer.DownloadBlobAsync(trainingParameters.CatalogFileRelativePath,
trainingFiles.CatalogFilePath, cancellationToken);
}
// download the evaluation files if provided
if (!string.IsNullOrWhiteSpace(trainingParameters.EvaluationUsageRelativePath))
{
// set local evaluation folder
trainingFiles.EvaluationUsageFolderPath = Path.Combine(localRootPath, EvaluationUsageLocalDirectoryName);
// create the local folder for evaluation usage events files
Directory.CreateDirectory(trainingFiles.EvaluationUsageFolderPath);
// check if the provided path represents a single file
if (await trainingBlobsContainer.ExistsAsync(trainingParameters.EvaluationUsageRelativePath,
cancellationToken))
{
string usageEventsBlobName = trainingParameters.EvaluationUsageRelativePath;
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath =
Path.Combine(trainingFiles.EvaluationUsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading evaluation usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
else
{
Trace.TraceInformation(
$"Listing all the evaluation usage events blobs under '{trainingParameters.EvaluationUsageRelativePath}'");
IList <string> evaluationUsageEventsBlobNames = await trainingBlobsContainer.ListBlobsAsync(
trainingParameters.EvaluationUsageRelativePath, cancellationToken);
Trace.TraceInformation(
$"Downloading all the evaluation usage events blobs (Found {evaluationUsageEventsBlobNames.Count})");
foreach (string usageEventsBlobName in evaluationUsageEventsBlobNames)
{
// set local usage events file path
string usageEventFileName = Path.GetFileName(usageEventsBlobName) ?? string.Empty;
string usageFilePath =
Path.Combine(trainingFiles.EvaluationUsageFolderPath, usageEventFileName);
Trace.TraceInformation($"Downloading evaluation usage events blob '{usageEventsBlobName}'");
await trainingBlobsContainer.DownloadBlobAsync(usageEventsBlobName, usageFilePath,
cancellationToken);
}
}
}
return(trainingFiles);
}
catch (StorageException storageException)
{
var exception = new Exception($"Failed downloading training files from storage. Model id: {modelId}",
storageException);
Trace.TraceError(exception.ToString());
throw exception;
}
}
