public virtual async Task UploadPartitionSuccessfulHashComputation(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var hashingOptions = new UploadTransactionalHashingOptions { Algorithm = algorithm }; // make pipeline assertion for checking hash was present on upload var hashPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestHashAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, options : clientOptions); // Act using (var stream = new MemoryStream(data)) { hashPipelineAssertion.CheckRequest = true; await UploadPartitionAsync(client, stream, hashingOptions); } // Assert // Assertion was in the pipeline and the service returning success means the hash was correct }
public virtual async Task UploadPartitionMismatchedHashThrows(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var hashingOptions = new UploadTransactionalHashingOptions { Algorithm = algorithm }; // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer var streamTamperPolicy = new TamperStreamContentsPolicy(); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(streamTamperPolicy, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, options : clientOptions); using (var stream = new MemoryStream(data)) { // Act streamTamperPolicy.TransformRequestBody = true; AsyncTestDelegate operation = async() => await UploadPartitionAsync(client, stream, hashingOptions); // Assert AssertWriteHashMismatch(operation, algorithm); } }
public override Task ParallelUploadOneShotSuccessfulHashComputation(TransactionalHashAlgorithm algorithm) { if (algorithm == TransactionalHashAlgorithm.StorageCrc64) { Assert.Inconclusive("Blob swagger currently doesn't support crc on PUT Blob"); } return(base.ParallelUploadOneShotSuccessfulHashComputation(algorithm)); }
private static void AssertSupportsHashAlgorithm(TransactionalHashAlgorithm algorithm) { if (algorithm == TransactionalHashAlgorithm.StorageCrc64) { /* Need to rerecord? Azure.Core framework won't record inconclusive tests. * Change this to pass for recording and revert when done. */ Assert.Inconclusive("Azure File Share does not support CRC64."); } }
public override Task ParallelUploadOneShotSuccessfulHashComputation(TransactionalHashAlgorithm algorithm) { if (algorithm == TransactionalHashAlgorithm.StorageCrc64) { /* Need to rerecord? Azure.Core framework won't record inconclusive tests. * Change this to pass for recording and revert when done. */ Assert.Inconclusive("Blob swagger currently doesn't support crc on PUT Blob"); } return(base.ParallelUploadOneShotSuccessfulHashComputation(algorithm)); }
public virtual async Task DownloadHashMismatchThrows( [Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm, [Values(true, false)] bool validate) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm, Validate = validate }; // alter response contents in pipeline, forcing a hash mismatch on verification step var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(new TamperStreamContentsPolicy() { TransformResponseBody = true }, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource : false, resourceName : resourceName, options : clientOptions); // Act AsyncTestDelegate operation = async() => await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length)); // Assert if (validate) { // SDK responsible for finding bad hash. Throw. ThrowsOrInconclusiveAsync <InvalidDataException>(operation); } else { // bad hash is for caller to find. Don't throw. await DoesNotThrowOrInconclusiveAsync(operation); } }
public virtual async Task OpenReadSuccessfulHashVerification( [Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm, [Values( // multiple reads that neatly align Constants.KB, // multiple reads with final having leftover buffer space 2 * Constants.KB, // buffer larger than data 4 * Constants.KB)] int bufferSize) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange // bufferSize/datasize MUST be a multiple of 512 for pageblob tests const int dataLength = 3 * Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); // make pipeline assertion for checking hash was present on download var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource : false, resourceName : resourceName, options : clientOptions); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm }; // Act var readStream = await OpenReadAsync(client, hashingOptions, bufferSize); // Assert hashPipelineAssertion.CheckResponse = true; await DoesNotThrowOrInconclusiveAsync(async() => await readStream.CopyToAsync(Stream.Null)); }
public virtual async Task ParallelDownloadSuccessfulHashVerification( [Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm, [Values(512, 2 * Constants.KB)] int chunkSize) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = 2 * Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); // make pipeline assertion for checking hash was present on download var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource : false, resourceName : resourceName, options : clientOptions); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm }; StorageTransferOptions transferOptions = new StorageTransferOptions { InitialTransferSize = chunkSize, MaximumTransferSize = chunkSize }; // Act hashPipelineAssertion.CheckResponse = true; await ParallelDownloadAsync(client, Stream.Null, hashingOptions, transferOptions); // Assert // Assertion was in the pipeline and the SDK not throwing means the hash was validated }
/// <summary> /// Asserts the computed hash matches the response content hash. /// </summary> /// <param name="computedHash">SDK computed hash.</param> /// <param name="algorithm">Hash algorithm identifier.</param> /// <param name="response">Response containing a response hash.</param> /// <exception cref="ArgumentException"> /// Throws if <paramref name="algorithm"/> is invalid. /// </exception> /// <exception cref="InvalidDataException"> /// Throws if the hashes do not match. /// </exception> private static void AssertResponseHashMatch(GetHashResult computedHash, TransactionalHashAlgorithm algorithm, Response response) { if (computedHash == default) { throw Errors.ArgumentNull(nameof(computedHash)); } if (response == default) { throw Errors.ArgumentNull(nameof(response)); } switch (algorithm) { case TransactionalHashAlgorithm.MD5: if (!Enumerable.SequenceEqual( computedHash.MD5, response.Headers.TryGetValue("Content-MD5", out byte[] md5) ? md5 : default))
/// <summary> /// Asserts the service returned an error that expected hash did not match hash on upload. /// </summary> /// <param name="writeAction">Async action to upload data to service.</param> /// <param name="algorithm">Hash algorithm used.</param> internal static void AssertWriteHashMismatch(AsyncTestDelegate writeAction, TransactionalHashAlgorithm algorithm) { var exception = ThrowsOrInconclusiveAsync <RequestFailedException>(writeAction); switch (algorithm) { case TransactionalHashAlgorithm.MD5: Assert.AreEqual("Md5Mismatch", exception.ErrorCode); break; case TransactionalHashAlgorithm.StorageCrc64: Assert.AreEqual("Crc64Mismatch", exception.ErrorCode); break; default: throw new ArgumentException("Test arguments contain bad algorithm specifier."); } }
public virtual async Task UploadPartitionUsePrecalculatedHash(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); // service throws different error for crc only when hash size in incorrect; we don't want to test that var hashSizeBytes = algorithm switch { TransactionalHashAlgorithm.MD5 => 16, TransactionalHashAlgorithm.StorageCrc64 => 8, _ => throw new ArgumentException("Cannot determine hash size for provided algorithm type") }; // hash needs to be wrong so we detect difference from auto-SDK correct calculation var precalculatedHash = GetRandomBuffer(hashSizeBytes); var hashingOptions = new UploadTransactionalHashingOptions { Algorithm = algorithm, PrecalculatedHash = precalculatedHash }; // make pipeline assertion for checking precalculated hash was present on upload var hashPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestHashAssertion(algorithm, expectedHash: precalculatedHash)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, options : clientOptions); hashPipelineAssertion.CheckRequest = true; using (var stream = new MemoryStream(data)) { // Act AsyncTestDelegate operation = async() => await UploadPartitionAsync(client, stream, hashingOptions); // Assert AssertWriteHashMismatch(operation, algorithm); } }
public virtual async Task DownloadSuccessfulHashVerification(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var resourceName = GetNewResourceName(); var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataLength, createResource : true, resourceName : resourceName); await SetupDataAsync(client, new MemoryStream(data)); var hashingOptions = new DownloadTransactionalHashingOptions { Algorithm = algorithm }; // Act var response = await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length)); // Assert // no policies this time; just check response headers switch (algorithm) { case TransactionalHashAlgorithm.MD5: Assert.True(response.Headers.Contains("Content-MD5")); break; case TransactionalHashAlgorithm.StorageCrc64: Assert.True(response.Headers.Contains("x-ms-content-crc64")); break; default: Assert.Fail("Test can't validate given algorithm type."); break; } }
/// <summary> /// Gets an assertion as to whether a transactional hash appeared on a returned response. /// Meant to be injected into a pipeline. /// </summary> /// <param name="algorithm"> /// Hash algorithm to look for. /// </param> /// <param name="isHashExpected"> /// Predicate to determine wheter a hash is expected on that particular response. E.g. on OpenRead, /// the initial GetProperties is not expected to have a hash, but download responses are. /// Defaults to all requests expected to have the hash. /// </param> /// <param name="expectedHash"> /// The actual hash value expected to be on the response, if known. Defaults to no specific value expected or checked. /// </param> /// <returns>An assertion to put into a pipeline policy.</returns> internal static Action <Response> GetResponseHashAssertion(TransactionalHashAlgorithm algorithm, Func <Response, bool> isHashExpected = default, byte[] expectedHash = default) { // action to assert a response header is as expected void AssertHash(ResponseHeaders headers, string headerName) { if (headers.TryGetValue(headerName, out string hash)) { if (expectedHash != default) { Assert.AreEqual(Convert.ToBase64String(expectedHash), hash); } } else { Assert.Fail($"{headerName} expected on response but was not found."); } }; return(response => { // filter some requests out with predicate if (isHashExpected != default && !isHashExpected(response)) { return; } switch (algorithm) { case TransactionalHashAlgorithm.MD5: AssertHash(response.Headers, "Content-MD5"); break; case TransactionalHashAlgorithm.StorageCrc64: AssertHash(response.Headers, "x-ms-content-crc64"); break; default: throw new Exception("Bad TransactionalHashAlgorithm provided to Response hash assertion."); } }); }
public async Task RoundtripWIthDefaults() { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const TransactionalHashAlgorithm expectedAlgorithm = TransactionalHashAlgorithm.StorageCrc64; const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var uploadHashingOptions = new UploadTransactionalHashingOptions(); var downloadHashingOptions = new DownloadTransactionalHashingOptions(); var clientOptions = ClientBuilder.GetOptions(); StorageTransferOptions transferOptions = new StorageTransferOptions { InitialTransferSize = 512, MaximumTransferSize = 512 }; // make pipeline assertion for checking hash was present on upload AND download var hashPipelineAssertion = new AssertMessageContentsPolicy( checkRequest: GetRequestHashAssertion(expectedAlgorithm, isHashExpected: ParallelUploadIsHashExpected), checkResponse: GetResponseHashAssertion(expectedAlgorithm)); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync(disposingContainer.Container, resourceLength : dataLength, createResource : true, options : clientOptions); // Act using (var stream = new MemoryStream(data)) { hashPipelineAssertion.CheckRequest = true; await ParallelUploadAsync(client, stream, uploadHashingOptions, transferOptions); hashPipelineAssertion.CheckRequest = false; } hashPipelineAssertion.CheckResponse = true; await ParallelDownloadAsync(client, Stream.Null, downloadHashingOptions, transferOptions); // Assert // Assertion was in the pipeline and the service returning success means the hash was correct }
public virtual async Task OpenWriteMismatchedHashThrows(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int streamBufferSize = Constants.KB; // this one needs to be 512 multiple for page blobs const int dataSize = Constants.KB - 11; // odd number to get some variance const int streamWrites = 10; var data = GetRandomBuffer(dataSize); var hashingOptions = new UploadTransactionalHashingOptions { Algorithm = algorithm }; // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer var clientOptions = ClientBuilder.GetOptions(); var tamperPolicy = new TamperStreamContentsPolicy(); clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( disposingContainer.Container, // should use dataSize instead of streamBufferSize but this gives 512 multiple and ends up irrelevant for this test resourceLength : streamBufferSize *streamWrites, createResource : true, options : clientOptions); // Act var writeStream = await OpenWriteAsync(client, hashingOptions, streamBufferSize); // Assert AssertWriteHashMismatch(async() => { tamperPolicy.TransformRequestBody = true; foreach (var _ in Enumerable.Range(0, streamWrites)) { await writeStream.WriteAsync(data, 0, data.Length); } }, algorithm); }
public virtual async Task PrecalculatedHashNotAccepted(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var hashingOptions = new UploadTransactionalHashingOptions { Algorithm = algorithm, PrecalculatedHash = GetRandomBuffer(16) }; var client = await GetResourceClientAsync(disposingContainer.Container, dataLength); // Act var exception = ThrowsOrInconclusiveAsync <ArgumentException>( async() => await ParallelUploadAsync(client, new MemoryStream(data), hashingOptions, transferOptions: default)); // Assert Assert.AreEqual("Precalculated hash not supported when potentially partitioning an upload.", exception.Message); }
public virtual async Task OpenWriteSuccessfulHashComputation(TransactionalHashAlgorithm algorithm) { await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync(); // Arrange const int streamBufferSize = Constants.KB; // this one needs to be 512 multiple for page blobs const int dataSize = Constants.KB - 11; // odd number to get some variance const int streamWrites = 10; var data = GetRandomBuffer(dataSize); var hashingOptions = new UploadTransactionalHashingOptions { Algorithm = algorithm }; // make pipeline assertion for checking hash was present on upload var hashPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestHashAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( disposingContainer.Container, // should use dataSize instead of streamBufferSize but this gives 512 multiple and ends up irrelevant for this test resourceLength : streamBufferSize *streamWrites, createResource : true, options : clientOptions); // Act var writeStream = await OpenWriteAsync(client, hashingOptions, streamBufferSize); // Assert hashPipelineAssertion.CheckRequest = true; foreach (var _ in Enumerable.Range(0, streamWrites)) { // triggers pipeline assertion await writeStream.WriteAsync(data, 0, data.Length); } }
public async Task ExpectedDownloadStreamingStreamTypeReturned(TransactionalHashAlgorithm algorithm, bool isBuffered) { await using var test = await GetDisposingContainerAsync(); // Arrange var data = GetRandomBuffer(Constants.KB); BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); using (var stream = new MemoryStream(data)) { await blob.UploadAsync(stream); } // don't make options instance at all for no hash request DownloadTransactionalHashingOptions hashingOptions = algorithm == TransactionalHashAlgorithm.None ? default : new DownloadTransactionalHashingOptions { Algorithm = algorithm }; // Act Response <BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(new BlobDownloadOptions { TransactionalHashingOptions = hashingOptions, Range = new HttpRange(length: data.Length) }); // Assert if (isBuffered) { Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType()); } // actual unbuffered stream type is private; just check we didn't get back a buffered stream else { Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType()); } }
public async Task HashingAndClientSideEncryptionIncompatible(TransactionalHashAlgorithm algorithm) { await using var disposingContainer = await GetDisposingContainerAsync(); // Arrange const int dataSize = Constants.KB; var data = GetRandomBuffer(dataSize); var hashingOptions = new UploadTransactionalHashingOptions { Algorithm = algorithm }; var encryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = new Mock <Core.Cryptography.IKeyEncryptionKey>().Object, KeyWrapAlgorithm = "foo" }; var clientOptions = ClientBuilder.GetOptions(); clientOptions._clientSideEncryptionOptions = encryptionOptions; var client = await GetResourceClientAsync( disposingContainer.Container, resourceLength : dataSize, createResource : true, options : clientOptions); // Act using var stream = new MemoryStream(data); var exception = Assert.ThrowsAsync <ArgumentException>(async() => await ParallelUploadAsync(client, stream, hashingOptions, transferOptions: default)); Assert.AreEqual("Client-side encryption and transactional hashing are not supported at the same time.", exception.Message); }
/// <summary> /// Asserts the content of the given array match the response content hash. /// </summary> /// <param name="content">Content to hash.</param> /// <param name="offset">Offset to start reading content at.</param> /// <param name="count">Number of bytes to read starting from the offset.</param> /// <param name="algorithm">Hash algorithm identifier.</param> /// <param name="response">Response containing a response hash.</param> /// <exception cref="ArgumentException"> /// Throws if <paramref name="algorithm"/> is invalid. /// </exception> /// <exception cref="InvalidDataException"> /// Throws if the hashes do not match. /// </exception> public static void AssertResponseHashMatch(byte[] content, int offset, int count, TransactionalHashAlgorithm algorithm, Response response) { GetHashResult computedHash = GetHash(content, offset, count, algorithm); AssertResponseHashMatch(computedHash, algorithm, response); }
/// <summary> /// Asserts the content of the given stream match the response content hash. /// </summary> /// <param name="content">Content to hash.</param> /// <param name="algorithm">Hash algorithm identifier.</param> /// <param name="response">Response containing a response hash.</param> /// <exception cref="ArgumentException"> /// Throws if <paramref name="algorithm"/> is invalid. /// </exception> /// <exception cref="InvalidDataException"> /// Throws if the hashes do not match. /// </exception> public static void AssertResponseHashMatch(Stream content, TransactionalHashAlgorithm algorithm, Response response) { GetHashResult computedHash = GetHash(content, algorithm); AssertResponseHashMatch(computedHash, algorithm, response); }