public PartitionedDownloader(
BlobBaseClient client,
StorageTransferOptions transferOptions = default,
DownloadTransactionalHashingOptions hashingOptions = default,
IProgress <long> progress = default)
{
_client = client;
// Set _maxWorkerCount
if (transferOptions.MaximumConcurrency.HasValue &&
transferOptions.MaximumConcurrency > 0)
{
_maxWorkerCount = transferOptions.MaximumConcurrency.Value;
}
else
{
_maxWorkerCount = Constants.Blob.Block.DefaultConcurrentTransfersCount;
}
// Set _rangeSize
if (transferOptions.MaximumTransferSize.HasValue &&
transferOptions.MaximumTransferSize.Value > 0)
{
_rangeSize = Math.Min(transferOptions.MaximumTransferSize.Value, Constants.Blob.Block.MaxDownloadBytes);
}
else
{
_rangeSize = Constants.DefaultBufferSize;
}
// Set _initialRangeSize
if (transferOptions.InitialTransferSize.HasValue &&
transferOptions.InitialTransferSize.Value > 0)
{
_initialRangeSize = transferOptions.InitialTransferSize.Value;
}
else
{
_initialRangeSize = Constants.Blob.Block.DefaultInitalDownloadRangeSize;
}
// the caller to this stream cannot defer validation, as they cannot access a returned hash
if (!(hashingOptions?.Validate ?? true))
{
throw Errors.CannotDeferTransactionalHashVerification();
}
_hashingOptions = hashingOptions;
_progress = progress;
/* Unlike partitioned upload, download cannot tell ahead of time if it will split and/or parallelize
* after first call. Instead of applying progress handling to initial download stream after-the-fact,
* wrap a given progress handler in an aggregator upfront and accept the overhead. */
if (_progress != null && _progress is not AggregatingProgressIncrementer)
{
_progress = new AggregatingProgressIncrementer(_progress);
}
}
protected override async Task <Stream> OpenReadAsync(
TBlobClient client,
DownloadTransactionalHashingOptions hashingOptions,
int internalBufferSize)
=> await client.OpenReadAsync(new BlobOpenReadOptions(false)
{
BufferSize = internalBufferSize,
TransactionalHashingOptions = hashingOptions
});
protected override async Task ParallelDownloadAsync(
TBlobClient client,
Stream destination,
DownloadTransactionalHashingOptions hashingOptions,
StorageTransferOptions transferOptions)
=> await client.DownloadToAsync(destination, new BlobDownloadToOptions
{
TransactionalHashingOptions = hashingOptions,
TransferOptions = transferOptions,
});
protected override async Task <Stream> OpenReadAsync(ShareFileClient client, DownloadTransactionalHashingOptions hashingOptions, int internalBufferSize)
{
AssertSupportsHashAlgorithm(hashingOptions?.Algorithm ?? default);
return(await client.OpenReadAsync(new ShareFileOpenReadOptions(false)
{
TransactionalHashingOptions = hashingOptions,
BufferSize = internalBufferSize
}));
}
public void TestDefaults()
{
var uploadOptions = new UploadTransactionalHashingOptions();
Assert.AreEqual(TransactionalHashAlgorithm.StorageCrc64, uploadOptions.Algorithm);
Assert.IsNull(uploadOptions.PrecalculatedHash);
var downloadOptions = new DownloadTransactionalHashingOptions();
Assert.AreEqual(TransactionalHashAlgorithm.StorageCrc64, downloadOptions.Algorithm);
Assert.IsTrue(downloadOptions.Validate);
}
public virtual async Task DownloadHashMismatchThrows(
[Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm,
[Values(true, false)] bool validate)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm, Validate = validate
};
// alter response contents in pipeline, forcing a hash mismatch on verification step
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(new TamperStreamContentsPolicy()
{
TransformResponseBody = true
}, HttpPipelinePosition.PerCall);
client = await GetResourceClientAsync(
disposingContainer.Container,
createResource : false,
resourceName : resourceName,
options : clientOptions);
// Act
AsyncTestDelegate operation = async() => await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length));
// Assert
if (validate)
{
// SDK responsible for finding bad hash. Throw.
ThrowsOrInconclusiveAsync <InvalidDataException>(operation);
}
else
{
// bad hash is for caller to find. Don't throw.
await DoesNotThrowOrInconclusiveAsync(operation);
}
}
public virtual async Task OpenReadSuccessfulHashVerification(
[Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm,
[Values(
// multiple reads that neatly align
Constants.KB,
// multiple reads with final having leftover buffer space
2 * Constants.KB,
// buffer larger than data
4 * Constants.KB)] int bufferSize)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
// bufferSize/datasize MUST be a multiple of 512 for pageblob tests
const int dataLength = 3 * Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
// make pipeline assertion for checking hash was present on download
var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm));
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
client = await GetResourceClientAsync(
disposingContainer.Container,
createResource : false,
resourceName : resourceName,
options : clientOptions);
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
// Act
var readStream = await OpenReadAsync(client, hashingOptions, bufferSize);
// Assert
hashPipelineAssertion.CheckResponse = true;
await DoesNotThrowOrInconclusiveAsync(async() => await readStream.CopyToAsync(Stream.Null));
}
/// <summary>
/// Deep copy constructor.
/// </summary>
/// <param name="deepCopySource"></param>
private BlobDownloadOptions(BlobDownloadOptions deepCopySource)
{
Argument.AssertNotNull(deepCopySource, nameof(deepCopySource));
Range = new HttpRange(offset: deepCopySource.Range.Offset, length: deepCopySource.Range.Length);
Conditions = BlobRequestConditions.CloneOrDefault(deepCopySource.Conditions);
// can't access an internal deep copy in Storage.Common
TransactionalHashingOptions = deepCopySource.TransactionalHashingOptions == default
? default
: new DownloadTransactionalHashingOptions()
{
Algorithm = deepCopySource.TransactionalHashingOptions.Algorithm,
Validate = deepCopySource.TransactionalHashingOptions.Validate
};
}
public PartitionedDownloader(
BlobBaseClient client,
StorageTransferOptions transferOptions = default,
DownloadTransactionalHashingOptions hashingOptions = default)
{
_client = client;
// Set _maxWorkerCount
if (transferOptions.MaximumConcurrency.HasValue &&
transferOptions.MaximumConcurrency > 0)
{
_maxWorkerCount = transferOptions.MaximumConcurrency.Value;
}
else
{
_maxWorkerCount = Constants.Blob.Block.DefaultConcurrentTransfersCount;
}
// Set _rangeSize
if (transferOptions.MaximumTransferSize.HasValue &&
transferOptions.MaximumTransferSize.Value > 0)
{
_rangeSize = Math.Min(transferOptions.MaximumTransferSize.Value, Constants.Blob.Block.MaxDownloadBytes);
}
else
{
_rangeSize = Constants.DefaultBufferSize;
}
// Set _initialRangeSize
if (transferOptions.InitialTransferSize.HasValue &&
transferOptions.InitialTransferSize.Value > 0)
{
_initialRangeSize = transferOptions.InitialTransferSize.Value;
}
else
{
_initialRangeSize = Constants.Blob.Block.DefaultInitalDownloadRangeSize;
}
// the caller to this stream cannot defer validation, as they cannot access a returned hash
if (!(hashingOptions?.Validate ?? true))
{
throw Errors.CannotDeferTransactionalHashVerification();
}
_hashingOptions = hashingOptions;
}
protected override async Task <Response> DownloadPartitionAsync(
TBlobClient client,
Stream destination,
DownloadTransactionalHashingOptions hashingOptions,
HttpRange range = default)
{
var response = await client.DownloadStreamingAsync(new BlobDownloadOptions
{
TransactionalHashingOptions = hashingOptions,
Range = range
});
await response.Value.Content.CopyToAsync(destination);
return(response.GetRawResponse());
}
public virtual async Task ParallelDownloadSuccessfulHashVerification(
[Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm,
[Values(512, 2 * Constants.KB)] int chunkSize)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = 2 * Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
// make pipeline assertion for checking hash was present on download
var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm));
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
client = await GetResourceClientAsync(
disposingContainer.Container,
createResource : false,
resourceName : resourceName,
options : clientOptions);
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
StorageTransferOptions transferOptions = new StorageTransferOptions
{
InitialTransferSize = chunkSize,
MaximumTransferSize = chunkSize
};
// Act
hashPipelineAssertion.CheckResponse = true;
await ParallelDownloadAsync(client, Stream.Null, hashingOptions, transferOptions);
// Assert
// Assertion was in the pipeline and the SDK not throwing means the hash was validated
}
public async Task RoundtripWIthDefaults()
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const TransactionalHashAlgorithm expectedAlgorithm = TransactionalHashAlgorithm.StorageCrc64;
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var uploadHashingOptions = new UploadTransactionalHashingOptions();
var downloadHashingOptions = new DownloadTransactionalHashingOptions();
var clientOptions = ClientBuilder.GetOptions();
StorageTransferOptions transferOptions = new StorageTransferOptions
{
InitialTransferSize = 512,
MaximumTransferSize = 512
};
// make pipeline assertion for checking hash was present on upload AND download
var hashPipelineAssertion = new AssertMessageContentsPolicy(
checkRequest: GetRequestHashAssertion(expectedAlgorithm, isHashExpected: ParallelUploadIsHashExpected),
checkResponse: GetResponseHashAssertion(expectedAlgorithm));
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
var client = await GetResourceClientAsync(disposingContainer.Container, resourceLength : dataLength, createResource : true, options : clientOptions);
// Act
using (var stream = new MemoryStream(data))
{
hashPipelineAssertion.CheckRequest = true;
await ParallelUploadAsync(client, stream, uploadHashingOptions, transferOptions);
hashPipelineAssertion.CheckRequest = false;
}
hashPipelineAssertion.CheckResponse = true;
await ParallelDownloadAsync(client, Stream.Null, downloadHashingOptions, transferOptions);
// Assert
// Assertion was in the pipeline and the service returning success means the hash was correct
}
public virtual async Task DownloadSuccessfulHashVerification(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
// Act
var response = await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length));
// Assert
// no policies this time; just check response headers
switch (algorithm)
{
case TransactionalHashAlgorithm.MD5:
Assert.True(response.Headers.Contains("Content-MD5"));
break;
case TransactionalHashAlgorithm.StorageCrc64:
Assert.True(response.Headers.Contains("x-ms-content-crc64"));
break;
default:
Assert.Fail("Test can't validate given algorithm type.");
break;
}
}
public async Task ExpectedDownloadStreamingStreamTypeReturned(TransactionalHashAlgorithm algorithm, bool isBuffered)
{
await using var test = await GetDisposingContainerAsync();
// Arrange
var data = GetRandomBuffer(Constants.KB);
BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName()));
using (var stream = new MemoryStream(data))
{
await blob.UploadAsync(stream);
}
// don't make options instance at all for no hash request
DownloadTransactionalHashingOptions hashingOptions = algorithm == TransactionalHashAlgorithm.None
? default
: new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
// Act
Response <BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(new BlobDownloadOptions
{
TransactionalHashingOptions = hashingOptions,
Range = new HttpRange(length: data.Length)
});
// Assert
if (isBuffered)
{
Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType());
}
// actual unbuffered stream type is private; just check we didn't get back a buffered stream
else
{
Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType());
}
}
protected override Task ParallelDownloadAsync(ShareFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, StorageTransferOptions transferOptions)
{
AssertSupportsHashAlgorithm(hashingOptions?.Algorithm ?? default);
/* Need to rerecord? Azure.Core framework won't record inconclusive tests.
* Change this to pass for recording and revert when done. */
Assert.Pass("Share file client does not support parallel download.");
return(Task.CompletedTask);
}
protected override async Task <Response> DownloadPartitionAsync(ShareFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, HttpRange range = default)
{
AssertSupportsHashAlgorithm(hashingOptions?.Algorithm ?? default);
var response = await client.DownloadAsync(new ShareFileDownloadOptions
{
TransactionalHashingOptions = hashingOptions,
Range = range
});
await response.Value.Content.CopyToAsync(destination);
return(response.GetRawResponse());
}
/// <summary>
/// Calls the parallel download method for the given resource client.
/// </summary>
/// <param name="client">Client to call download on.</param>
/// <param name="destination">Where to send downloaded data.</param>
/// <param name="hashingOptions">Transactional hashing options to use on download.</param>
/// <param name="transferOptions">Storage transfer options to use on download.</param>
protected abstract Task ParallelDownloadAsync(
TResourceClient client,
Stream destination,
DownloadTransactionalHashingOptions hashingOptions,
StorageTransferOptions transferOptions);
/// <summary>
/// Calls the open read method for the given resource client.
/// </summary>
/// <param name="client">Client to call open read on.</param>
/// <param name="hashingOptions">Transactinal hashing options to use in the read stream.</param>
/// <param name="internalBufferSize">Buffer size for the read stream.</param>
protected abstract Task <Stream> OpenReadAsync(
TResourceClient client,
DownloadTransactionalHashingOptions hashingOptions,
int internalBufferSize);
protected override async Task <Stream> OpenReadAsync(DataLakeFileClient client, DownloadTransactionalHashingOptions hashingOptions, int internalBufferSize)
{
return(await client.OpenReadAsync(new DataLakeOpenReadOptions(false)
{
TransactionalHashingOptions = hashingOptions,
BufferSize = internalBufferSize
}));
}
protected override async Task ParallelDownloadAsync(DataLakeFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, StorageTransferOptions transferOptions)
{
await client.ReadToAsync(destination, new DataLakeFileReadToOptions
{
TransactionalHashingOptions = hashingOptions,
TransferOptions = transferOptions
});
}
/// <summary>
/// Calls the 1:1 download method for the given resource client.
/// </summary>
/// <param name="client">Client to call the download on.</param>
/// <param name="destination">Where to send downloaded data.</param>
/// <param name="hashingOptions">Transactional hashing options to use on download.</param>
/// <param name="range">Range parameter for download, necessary for transactional hash request to be accepted by service.</param>
protected abstract Task <Response> DownloadPartitionAsync(
TResourceClient client,
Stream destination,
DownloadTransactionalHashingOptions hashingOptions,
HttpRange range = default);
