public PartitionedDownloader( BlobBaseClient client, StorageTransferOptions transferOptions = default, DownloadTransactionalHashingOptions hashingOptions = default, IProgress <long> progress = default) { _client = client; // Set _maxWorkerCount if (transferOptions.MaximumConcurrency.HasValue && transferOptions.MaximumConcurrency > 0) { _maxWorkerCount = transferOptions.MaximumConcurrency.Value; } else { _maxWorkerCount = Constants.Blob.Block.DefaultConcurrentTransfersCount; } // Set _rangeSize if (transferOptions.MaximumTransferSize.HasValue && transferOptions.MaximumTransferSize.Value > 0) { _rangeSize = Math.Min(transferOptions.MaximumTransferSize.Value, Constants.Blob.Block.MaxDownloadBytes); } else { _rangeSize = Constants.DefaultBufferSize; } // Set _initialRangeSize if (transferOptions.InitialTransferSize.HasValue && transferOptions.InitialTransferSize.Value > 0) { _initialRangeSize = transferOptions.InitialTransferSize.Value; } else { _initialRangeSize = Constants.Blob.Block.DefaultInitalDownloadRangeSize; } // the caller to this stream cannot defer validation, as they cannot access a returned hash if (!(hashingOptions?.Validate ?? true)) { throw Errors.CannotDeferTransactionalHashVerification(); } _hashingOptions = hashingOptions; _progress = progress; /* Unlike partitioned upload, download cannot tell ahead of time if it will split and/or parallelize * after first call. Instead of applying progress handling to initial download stream after-the-fact, * wrap a given progress handler in an aggregator upfront and accept the overhead. */ if (_progress != null && _progress is not AggregatingProgressIncrementer) { _progress = new AggregatingProgressIncrementer(_progress); } }
protected override async Task <Stream> OpenReadAsync( TBlobClient client, DownloadTransactionalHashingOptions hashingOptions, int internalBufferSize) => await client.OpenReadAsync(new BlobOpenReadOptions(false) { BufferSize = internalBufferSize, TransactionalHashingOptions = hashingOptions });
protected override async Task ParallelDownloadAsync( TBlobClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, StorageTransferOptions transferOptions) => await client.DownloadToAsync(destination, new BlobDownloadToOptions { TransactionalHashingOptions = hashingOptions, TransferOptions = transferOptions, });
protected override async Task <Stream> OpenReadAsync(ShareFileClient client, DownloadTransactionalHashingOptions hashingOptions, int internalBufferSize) { AssertSupportsHashAlgorithm(hashingOptions?.Algorithm ?? default); return(await client.OpenReadAsync(new ShareFileOpenReadOptions(false) { TransactionalHashingOptions = hashingOptions, BufferSize = internalBufferSize })); }
public void TestDefaults() { var uploadOptions = new UploadTransactionalHashingOptions(); Assert.AreEqual(TransactionalHashAlgorithm.StorageCrc64, uploadOptions.Algorithm); Assert.IsNull(uploadOptions.PrecalculatedHash); var downloadOptions = new DownloadTransactionalHashingOptions(); Assert.AreEqual(TransactionalHashAlgorithm.StorageCrc64, downloadOptions.Algorithm); Assert.IsTrue(downloadOptions.Validate); }
public virtual async Task DownloadHashMismatchThrows( [Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm, [Values(true, false)] bool validate) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm, Validate = validate }; // alter response contents in pipeline, forcing a hash mismatch on verification step var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(new TamperStreamContentsPolicy() { TransformResponseBody = true }, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource : false, resourceName : resourceName, options : clientOptions); // Act AsyncTestDelegate operation = async() => await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length)); // Assert if (validate) { // SDK responsible for finding bad hash. Throw. ThrowsOrInconclusiveAsync <InvalidDataException>(operation); } else { // bad hash is for caller to find. Don't throw. await DoesNotThrowOrInconclusiveAsync(operation); } }
public virtual async Task OpenReadSuccessfulHashVerification( [Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm, [Values( // multiple reads that neatly align Constants.KB, // multiple reads with final having leftover buffer space 2 * Constants.KB, // buffer larger than data 4 * Constants.KB)] int bufferSize) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange // bufferSize/datasize MUST be a multiple of 512 for pageblob tests const int dataLength = 3 * Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); // make pipeline assertion for checking hash was present on download var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource : false, resourceName : resourceName, options : clientOptions); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm }; // Act var readStream = await OpenReadAsync(client, hashingOptions, bufferSize); // Assert hashPipelineAssertion.CheckResponse = true; await DoesNotThrowOrInconclusiveAsync(async() => await readStream.CopyToAsync(Stream.Null)); }
/// <summary> /// Deep copy constructor. /// </summary> /// <param name="deepCopySource"></param> private BlobDownloadOptions(BlobDownloadOptions deepCopySource) { Argument.AssertNotNull(deepCopySource, nameof(deepCopySource)); Range = new HttpRange(offset: deepCopySource.Range.Offset, length: deepCopySource.Range.Length); Conditions = BlobRequestConditions.CloneOrDefault(deepCopySource.Conditions); // can't access an internal deep copy in Storage.Common TransactionalHashingOptions = deepCopySource.TransactionalHashingOptions == default ? default : new DownloadTransactionalHashingOptions() { Algorithm = deepCopySource.TransactionalHashingOptions.Algorithm, Validate = deepCopySource.TransactionalHashingOptions.Validate }; }
public PartitionedDownloader( BlobBaseClient client, StorageTransferOptions transferOptions = default, DownloadTransactionalHashingOptions hashingOptions = default) { _client = client; // Set _maxWorkerCount if (transferOptions.MaximumConcurrency.HasValue && transferOptions.MaximumConcurrency > 0) { _maxWorkerCount = transferOptions.MaximumConcurrency.Value; } else { _maxWorkerCount = Constants.Blob.Block.DefaultConcurrentTransfersCount; } // Set _rangeSize if (transferOptions.MaximumTransferSize.HasValue && transferOptions.MaximumTransferSize.Value > 0) { _rangeSize = Math.Min(transferOptions.MaximumTransferSize.Value, Constants.Blob.Block.MaxDownloadBytes); } else { _rangeSize = Constants.DefaultBufferSize; } // Set _initialRangeSize if (transferOptions.InitialTransferSize.HasValue && transferOptions.InitialTransferSize.Value > 0) { _initialRangeSize = transferOptions.InitialTransferSize.Value; } else { _initialRangeSize = Constants.Blob.Block.DefaultInitalDownloadRangeSize; } // the caller to this stream cannot defer validation, as they cannot access a returned hash if (!(hashingOptions?.Validate ?? true)) { throw Errors.CannotDeferTransactionalHashVerification(); } _hashingOptions = hashingOptions; }
protected override async Task <Response> DownloadPartitionAsync( TBlobClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, HttpRange range = default) { var response = await client.DownloadStreamingAsync(new BlobDownloadOptions { TransactionalHashingOptions = hashingOptions, Range = range }); await response.Value.Content.CopyToAsync(destination); return(response.GetRawResponse()); }
public virtual async Task ParallelDownloadSuccessfulHashVerification( [Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm, [Values(512, 2 * Constants.KB)] int chunkSize) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = 2 * Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); // make pipeline assertion for checking hash was present on download var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource : false, resourceName : resourceName, options : clientOptions); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm }; StorageTransferOptions transferOptions = new StorageTransferOptions { InitialTransferSize = chunkSize, MaximumTransferSize = chunkSize }; // Act hashPipelineAssertion.CheckResponse = true; await ParallelDownloadAsync(client, Stream.Null, hashingOptions, transferOptions); // Assert // Assertion was in the pipeline and the SDK not throwing means the hash was validated }
public async Task RoundtripWIthDefaults() { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const TransactionalHashAlgorithm expectedAlgorithm = TransactionalHashAlgorithm.StorageCrc64; const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var uploadHashingOptions = new UploadTransactionalHashingOptions(); var downloadHashingOptions = new DownloadTransactionalHashingOptions(); var clientOptions = ClientBuilder.GetOptions(); StorageTransferOptions transferOptions = new StorageTransferOptions { InitialTransferSize = 512, MaximumTransferSize = 512 }; // make pipeline assertion for checking hash was present on upload AND download var hashPipelineAssertion = new AssertMessageContentsPolicy( checkRequest: GetRequestHashAssertion(expectedAlgorithm, isHashExpected: ParallelUploadIsHashExpected), checkResponse: GetResponseHashAssertion(expectedAlgorithm)); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync(disposingContainer.Container, resourceLength : dataLength, createResource : true, options : clientOptions); // Act using (var stream = new MemoryStream(data)) { hashPipelineAssertion.CheckRequest = true; await ParallelUploadAsync(client, stream, uploadHashingOptions, transferOptions); hashPipelineAssertion.CheckRequest = false; } hashPipelineAssertion.CheckResponse = true; await ParallelDownloadAsync(client, Stream.Null, downloadHashingOptions, transferOptions); // Assert // Assertion was in the pipeline and the service returning success means the hash was correct }
public virtual async Task DownloadSuccessfulHashVerification(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm }; // Act var response = await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length)); // Assert // no policies this time; just check response headers switch (algorithm) { case TransactionalHashAlgorithm.MD5: Assert.True(response.Headers.Contains("Content-MD5")); break; case TransactionalHashAlgorithm.StorageCrc64: Assert.True(response.Headers.Contains("x-ms-content-crc64")); break; default: Assert.Fail("Test can't validate given algorithm type."); break; } }
public async Task ExpectedDownloadStreamingStreamTypeReturned(TransactionalHashAlgorithm algorithm, bool isBuffered) { await using var test = await GetDisposingContainerAsync(); // Arrange var data = GetRandomBuffer(Constants.KB); BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); using (var stream = new MemoryStream(data)) { await blob.UploadAsync(stream); } // don't make options instance at all for no hash request DownloadTransactionalHashingOptions hashingOptions = algorithm == TransactionalHashAlgorithm.None ? default : new DownloadTransactionalHashingOptions { Algorithm = algorithm }; // Act Response <BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(new BlobDownloadOptions { TransactionalHashingOptions = hashingOptions, Range = new HttpRange(length: data.Length) }); // Assert if (isBuffered) { Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType()); } // actual unbuffered stream type is private; just check we didn't get back a buffered stream else { Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType()); } }
protected override Task ParallelDownloadAsync(ShareFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, StorageTransferOptions transferOptions) { AssertSupportsHashAlgorithm(hashingOptions?.Algorithm ?? default); /* Need to rerecord? Azure.Core framework won't record inconclusive tests. * Change this to pass for recording and revert when done. */ Assert.Pass("Share file client does not support parallel download."); return(Task.CompletedTask); }
protected override async Task <Response> DownloadPartitionAsync(ShareFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, HttpRange range = default) { AssertSupportsHashAlgorithm(hashingOptions?.Algorithm ?? default); var response = await client.DownloadAsync(new ShareFileDownloadOptions { TransactionalHashingOptions = hashingOptions, Range = range }); await response.Value.Content.CopyToAsync(destination); return(response.GetRawResponse()); }
/// <summary> /// Calls the parallel download method for the given resource client. /// </summary> /// <param name="client">Client to call download on.</param> /// <param name="destination">Where to send downloaded data.</param> /// <param name="hashingOptions">Transactional hashing options to use on download.</param> /// <param name="transferOptions">Storage transfer options to use on download.</param> protected abstract Task ParallelDownloadAsync( TResourceClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, StorageTransferOptions transferOptions);
/// <summary> /// Calls the open read method for the given resource client. /// </summary> /// <param name="client">Client to call open read on.</param> /// <param name="hashingOptions">Transactinal hashing options to use in the read stream.</param> /// <param name="internalBufferSize">Buffer size for the read stream.</param> protected abstract Task <Stream> OpenReadAsync( TResourceClient client, DownloadTransactionalHashingOptions hashingOptions, int internalBufferSize);
protected override async Task <Stream> OpenReadAsync(DataLakeFileClient client, DownloadTransactionalHashingOptions hashingOptions, int internalBufferSize) { return(await client.OpenReadAsync(new DataLakeOpenReadOptions(false) { TransactionalHashingOptions = hashingOptions, BufferSize = internalBufferSize })); }
protected override async Task ParallelDownloadAsync(DataLakeFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, StorageTransferOptions transferOptions) { await client.ReadToAsync(destination, new DataLakeFileReadToOptions { TransactionalHashingOptions = hashingOptions, TransferOptions = transferOptions }); }
/// <summary> /// Calls the 1:1 download method for the given resource client. /// </summary> /// <param name="client">Client to call the download on.</param> /// <param name="destination">Where to send downloaded data.</param> /// <param name="hashingOptions">Transactional hashing options to use on download.</param> /// <param name="range">Range parameter for download, necessary for transactional hash request to be accepted by service.</param> protected abstract Task <Response> DownloadPartitionAsync( TResourceClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, HttpRange range = default);