public virtual async Task UploadPartitionSuccessfulHashComputation(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var hashingOptions = new UploadTransactionalHashingOptions
{
Algorithm = algorithm
};
// make pipeline assertion for checking hash was present on upload
var hashPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestHashAssertion(algorithm));
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
options : clientOptions);
// Act
using (var stream = new MemoryStream(data))
{
hashPipelineAssertion.CheckRequest = true;
await UploadPartitionAsync(client, stream, hashingOptions);
}
// Assert
// Assertion was in the pipeline and the service returning success means the hash was correct
}
public virtual async Task UploadPartitionMismatchedHashThrows(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var hashingOptions = new UploadTransactionalHashingOptions
{
Algorithm = algorithm
};
// Tamper with stream contents in the pipeline to simulate silent failure in the transit layer
var streamTamperPolicy = new TamperStreamContentsPolicy();
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(streamTamperPolicy, HttpPipelinePosition.PerCall);
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
options : clientOptions);
using (var stream = new MemoryStream(data))
{
// Act
streamTamperPolicy.TransformRequestBody = true;
AsyncTestDelegate operation = async() => await UploadPartitionAsync(client, stream, hashingOptions);
// Assert
AssertWriteHashMismatch(operation, algorithm);
}
}
public override Task ParallelUploadOneShotSuccessfulHashComputation(TransactionalHashAlgorithm algorithm)
{
if (algorithm == TransactionalHashAlgorithm.StorageCrc64)
{
Assert.Inconclusive("Blob swagger currently doesn't support crc on PUT Blob");
}
return(base.ParallelUploadOneShotSuccessfulHashComputation(algorithm));
}
private static void AssertSupportsHashAlgorithm(TransactionalHashAlgorithm algorithm)
{
if (algorithm == TransactionalHashAlgorithm.StorageCrc64)
{
/* Need to rerecord? Azure.Core framework won't record inconclusive tests.
* Change this to pass for recording and revert when done. */
Assert.Inconclusive("Azure File Share does not support CRC64.");
}
}
public override Task ParallelUploadOneShotSuccessfulHashComputation(TransactionalHashAlgorithm algorithm)
{
if (algorithm == TransactionalHashAlgorithm.StorageCrc64)
{
/* Need to rerecord? Azure.Core framework won't record inconclusive tests.
* Change this to pass for recording and revert when done. */
Assert.Inconclusive("Blob swagger currently doesn't support crc on PUT Blob");
}
return(base.ParallelUploadOneShotSuccessfulHashComputation(algorithm));
}
public virtual async Task DownloadHashMismatchThrows(
[Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm,
[Values(true, false)] bool validate)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm, Validate = validate
};
// alter response contents in pipeline, forcing a hash mismatch on verification step
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(new TamperStreamContentsPolicy()
{
TransformResponseBody = true
}, HttpPipelinePosition.PerCall);
client = await GetResourceClientAsync(
disposingContainer.Container,
createResource : false,
resourceName : resourceName,
options : clientOptions);
// Act
AsyncTestDelegate operation = async() => await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length));
// Assert
if (validate)
{
// SDK responsible for finding bad hash. Throw.
ThrowsOrInconclusiveAsync <InvalidDataException>(operation);
}
else
{
// bad hash is for caller to find. Don't throw.
await DoesNotThrowOrInconclusiveAsync(operation);
}
}
public virtual async Task OpenReadSuccessfulHashVerification(
[Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm,
[Values(
// multiple reads that neatly align
Constants.KB,
// multiple reads with final having leftover buffer space
2 * Constants.KB,
// buffer larger than data
4 * Constants.KB)] int bufferSize)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
// bufferSize/datasize MUST be a multiple of 512 for pageblob tests
const int dataLength = 3 * Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
// make pipeline assertion for checking hash was present on download
var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm));
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
client = await GetResourceClientAsync(
disposingContainer.Container,
createResource : false,
resourceName : resourceName,
options : clientOptions);
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
// Act
var readStream = await OpenReadAsync(client, hashingOptions, bufferSize);
// Assert
hashPipelineAssertion.CheckResponse = true;
await DoesNotThrowOrInconclusiveAsync(async() => await readStream.CopyToAsync(Stream.Null));
}
public virtual async Task ParallelDownloadSuccessfulHashVerification(
[Values(TransactionalHashAlgorithm.MD5, TransactionalHashAlgorithm.StorageCrc64)] TransactionalHashAlgorithm algorithm,
[Values(512, 2 * Constants.KB)] int chunkSize)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = 2 * Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
// make pipeline assertion for checking hash was present on download
var hashPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: GetResponseHashAssertion(algorithm));
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
client = await GetResourceClientAsync(
disposingContainer.Container,
createResource : false,
resourceName : resourceName,
options : clientOptions);
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
StorageTransferOptions transferOptions = new StorageTransferOptions
{
InitialTransferSize = chunkSize,
MaximumTransferSize = chunkSize
};
// Act
hashPipelineAssertion.CheckResponse = true;
await ParallelDownloadAsync(client, Stream.Null, hashingOptions, transferOptions);
// Assert
// Assertion was in the pipeline and the SDK not throwing means the hash was validated
}
/// <summary>
/// Asserts the computed hash matches the response content hash.
/// </summary>
/// <param name="computedHash">SDK computed hash.</param>
/// <param name="algorithm">Hash algorithm identifier.</param>
/// <param name="response">Response containing a response hash.</param>
/// <exception cref="ArgumentException">
/// Throws if <paramref name="algorithm"/> is invalid.
/// </exception>
/// <exception cref="InvalidDataException">
/// Throws if the hashes do not match.
/// </exception>
private static void AssertResponseHashMatch(GetHashResult computedHash, TransactionalHashAlgorithm algorithm, Response response)
{
if (computedHash == default)
{
throw Errors.ArgumentNull(nameof(computedHash));
}
if (response == default)
{
throw Errors.ArgumentNull(nameof(response));
}
switch (algorithm)
{
case TransactionalHashAlgorithm.MD5:
if (!Enumerable.SequenceEqual(
computedHash.MD5,
response.Headers.TryGetValue("Content-MD5", out byte[] md5) ? md5 : default))
/// <summary>
/// Asserts the service returned an error that expected hash did not match hash on upload.
/// </summary>
/// <param name="writeAction">Async action to upload data to service.</param>
/// <param name="algorithm">Hash algorithm used.</param>
internal static void AssertWriteHashMismatch(AsyncTestDelegate writeAction, TransactionalHashAlgorithm algorithm)
{
var exception = ThrowsOrInconclusiveAsync <RequestFailedException>(writeAction);
switch (algorithm)
{
case TransactionalHashAlgorithm.MD5:
Assert.AreEqual("Md5Mismatch", exception.ErrorCode);
break;
case TransactionalHashAlgorithm.StorageCrc64:
Assert.AreEqual("Crc64Mismatch", exception.ErrorCode);
break;
default:
throw new ArgumentException("Test arguments contain bad algorithm specifier.");
}
}
public virtual async Task UploadPartitionUsePrecalculatedHash(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
// service throws different error for crc only when hash size in incorrect; we don't want to test that
var hashSizeBytes = algorithm switch
{
TransactionalHashAlgorithm.MD5 => 16,
TransactionalHashAlgorithm.StorageCrc64 => 8,
_ => throw new ArgumentException("Cannot determine hash size for provided algorithm type")
};
// hash needs to be wrong so we detect difference from auto-SDK correct calculation
var precalculatedHash = GetRandomBuffer(hashSizeBytes);
var hashingOptions = new UploadTransactionalHashingOptions
{
Algorithm = algorithm,
PrecalculatedHash = precalculatedHash
};
// make pipeline assertion for checking precalculated hash was present on upload
var hashPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestHashAssertion(algorithm, expectedHash: precalculatedHash));
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
options : clientOptions);
hashPipelineAssertion.CheckRequest = true;
using (var stream = new MemoryStream(data))
{
// Act
AsyncTestDelegate operation = async() => await UploadPartitionAsync(client, stream, hashingOptions);
// Assert
AssertWriteHashMismatch(operation, algorithm);
}
}
public virtual async Task DownloadSuccessfulHashVerification(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var resourceName = GetNewResourceName();
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataLength,
createResource : true,
resourceName : resourceName);
await SetupDataAsync(client, new MemoryStream(data));
var hashingOptions = new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
// Act
var response = await DownloadPartitionAsync(client, Stream.Null, hashingOptions, new HttpRange(length : data.Length));
// Assert
// no policies this time; just check response headers
switch (algorithm)
{
case TransactionalHashAlgorithm.MD5:
Assert.True(response.Headers.Contains("Content-MD5"));
break;
case TransactionalHashAlgorithm.StorageCrc64:
Assert.True(response.Headers.Contains("x-ms-content-crc64"));
break;
default:
Assert.Fail("Test can't validate given algorithm type.");
break;
}
}
/// <summary>
/// Gets an assertion as to whether a transactional hash appeared on a returned response.
/// Meant to be injected into a pipeline.
/// </summary>
/// <param name="algorithm">
/// Hash algorithm to look for.
/// </param>
/// <param name="isHashExpected">
/// Predicate to determine wheter a hash is expected on that particular response. E.g. on OpenRead,
/// the initial GetProperties is not expected to have a hash, but download responses are.
/// Defaults to all requests expected to have the hash.
/// </param>
/// <param name="expectedHash">
/// The actual hash value expected to be on the response, if known. Defaults to no specific value expected or checked.
/// </param>
/// <returns>An assertion to put into a pipeline policy.</returns>
internal static Action <Response> GetResponseHashAssertion(TransactionalHashAlgorithm algorithm, Func <Response, bool> isHashExpected = default, byte[] expectedHash = default)
{
// action to assert a response header is as expected
void AssertHash(ResponseHeaders headers, string headerName)
{
if (headers.TryGetValue(headerName, out string hash))
{
if (expectedHash != default)
{
Assert.AreEqual(Convert.ToBase64String(expectedHash), hash);
}
}
else
{
Assert.Fail($"{headerName} expected on response but was not found.");
}
};
return(response =>
{
// filter some requests out with predicate
if (isHashExpected != default && !isHashExpected(response))
{
return;
}
switch (algorithm)
{
case TransactionalHashAlgorithm.MD5:
AssertHash(response.Headers, "Content-MD5");
break;
case TransactionalHashAlgorithm.StorageCrc64:
AssertHash(response.Headers, "x-ms-content-crc64");
break;
default:
throw new Exception("Bad TransactionalHashAlgorithm provided to Response hash assertion.");
}
});
}
public async Task RoundtripWIthDefaults()
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const TransactionalHashAlgorithm expectedAlgorithm = TransactionalHashAlgorithm.StorageCrc64;
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var uploadHashingOptions = new UploadTransactionalHashingOptions();
var downloadHashingOptions = new DownloadTransactionalHashingOptions();
var clientOptions = ClientBuilder.GetOptions();
StorageTransferOptions transferOptions = new StorageTransferOptions
{
InitialTransferSize = 512,
MaximumTransferSize = 512
};
// make pipeline assertion for checking hash was present on upload AND download
var hashPipelineAssertion = new AssertMessageContentsPolicy(
checkRequest: GetRequestHashAssertion(expectedAlgorithm, isHashExpected: ParallelUploadIsHashExpected),
checkResponse: GetResponseHashAssertion(expectedAlgorithm));
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
var client = await GetResourceClientAsync(disposingContainer.Container, resourceLength : dataLength, createResource : true, options : clientOptions);
// Act
using (var stream = new MemoryStream(data))
{
hashPipelineAssertion.CheckRequest = true;
await ParallelUploadAsync(client, stream, uploadHashingOptions, transferOptions);
hashPipelineAssertion.CheckRequest = false;
}
hashPipelineAssertion.CheckResponse = true;
await ParallelDownloadAsync(client, Stream.Null, downloadHashingOptions, transferOptions);
// Assert
// Assertion was in the pipeline and the service returning success means the hash was correct
}
public virtual async Task OpenWriteMismatchedHashThrows(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int streamBufferSize = Constants.KB; // this one needs to be 512 multiple for page blobs
const int dataSize = Constants.KB - 11; // odd number to get some variance
const int streamWrites = 10;
var data = GetRandomBuffer(dataSize);
var hashingOptions = new UploadTransactionalHashingOptions
{
Algorithm = algorithm
};
// Tamper with stream contents in the pipeline to simulate silent failure in the transit layer
var clientOptions = ClientBuilder.GetOptions();
var tamperPolicy = new TamperStreamContentsPolicy();
clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall);
var client = await GetResourceClientAsync(
disposingContainer.Container,
// should use dataSize instead of streamBufferSize but this gives 512 multiple and ends up irrelevant for this test
resourceLength : streamBufferSize *streamWrites,
createResource : true,
options : clientOptions);
// Act
var writeStream = await OpenWriteAsync(client, hashingOptions, streamBufferSize);
// Assert
AssertWriteHashMismatch(async() =>
{
tamperPolicy.TransformRequestBody = true;
foreach (var _ in Enumerable.Range(0, streamWrites))
{
await writeStream.WriteAsync(data, 0, data.Length);
}
}, algorithm);
}
public virtual async Task PrecalculatedHashNotAccepted(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataLength = Constants.KB;
var data = GetRandomBuffer(dataLength);
var hashingOptions = new UploadTransactionalHashingOptions
{
Algorithm = algorithm,
PrecalculatedHash = GetRandomBuffer(16)
};
var client = await GetResourceClientAsync(disposingContainer.Container, dataLength);
// Act
var exception = ThrowsOrInconclusiveAsync <ArgumentException>(
async() => await ParallelUploadAsync(client, new MemoryStream(data), hashingOptions, transferOptions: default));
// Assert
Assert.AreEqual("Precalculated hash not supported when potentially partitioning an upload.", exception.Message);
}
public virtual async Task OpenWriteSuccessfulHashComputation(TransactionalHashAlgorithm algorithm)
{
await using IDisposingContainer <TContainerClient> disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int streamBufferSize = Constants.KB; // this one needs to be 512 multiple for page blobs
const int dataSize = Constants.KB - 11; // odd number to get some variance
const int streamWrites = 10;
var data = GetRandomBuffer(dataSize);
var hashingOptions = new UploadTransactionalHashingOptions
{
Algorithm = algorithm
};
// make pipeline assertion for checking hash was present on upload
var hashPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestHashAssertion(algorithm));
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(hashPipelineAssertion, HttpPipelinePosition.PerCall);
var client = await GetResourceClientAsync(
disposingContainer.Container,
// should use dataSize instead of streamBufferSize but this gives 512 multiple and ends up irrelevant for this test
resourceLength : streamBufferSize *streamWrites,
createResource : true,
options : clientOptions);
// Act
var writeStream = await OpenWriteAsync(client, hashingOptions, streamBufferSize);
// Assert
hashPipelineAssertion.CheckRequest = true;
foreach (var _ in Enumerable.Range(0, streamWrites))
{
// triggers pipeline assertion
await writeStream.WriteAsync(data, 0, data.Length);
}
}
public async Task ExpectedDownloadStreamingStreamTypeReturned(TransactionalHashAlgorithm algorithm, bool isBuffered)
{
await using var test = await GetDisposingContainerAsync();
// Arrange
var data = GetRandomBuffer(Constants.KB);
BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName()));
using (var stream = new MemoryStream(data))
{
await blob.UploadAsync(stream);
}
// don't make options instance at all for no hash request
DownloadTransactionalHashingOptions hashingOptions = algorithm == TransactionalHashAlgorithm.None
? default
: new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
// Act
Response <BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(new BlobDownloadOptions
{
TransactionalHashingOptions = hashingOptions,
Range = new HttpRange(length: data.Length)
});
// Assert
if (isBuffered)
{
Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType());
}
// actual unbuffered stream type is private; just check we didn't get back a buffered stream
else
{
Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType());
}
}
public async Task HashingAndClientSideEncryptionIncompatible(TransactionalHashAlgorithm algorithm)
{
await using var disposingContainer = await GetDisposingContainerAsync();
// Arrange
const int dataSize = Constants.KB;
var data = GetRandomBuffer(dataSize);
var hashingOptions = new UploadTransactionalHashingOptions
{
Algorithm = algorithm
};
var encryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0)
{
KeyEncryptionKey = new Mock <Core.Cryptography.IKeyEncryptionKey>().Object,
KeyWrapAlgorithm = "foo"
};
var clientOptions = ClientBuilder.GetOptions();
clientOptions._clientSideEncryptionOptions = encryptionOptions;
var client = await GetResourceClientAsync(
disposingContainer.Container,
resourceLength : dataSize,
createResource : true,
options : clientOptions);
// Act
using var stream = new MemoryStream(data);
var exception = Assert.ThrowsAsync <ArgumentException>(async() => await ParallelUploadAsync(client, stream, hashingOptions, transferOptions: default));
Assert.AreEqual("Client-side encryption and transactional hashing are not supported at the same time.", exception.Message);
}
/// <summary>
/// Asserts the content of the given array match the response content hash.
/// </summary>
/// <param name="content">Content to hash.</param>
/// <param name="offset">Offset to start reading content at.</param>
/// <param name="count">Number of bytes to read starting from the offset.</param>
/// <param name="algorithm">Hash algorithm identifier.</param>
/// <param name="response">Response containing a response hash.</param>
/// <exception cref="ArgumentException">
/// Throws if <paramref name="algorithm"/> is invalid.
/// </exception>
/// <exception cref="InvalidDataException">
/// Throws if the hashes do not match.
/// </exception>
public static void AssertResponseHashMatch(byte[] content, int offset, int count, TransactionalHashAlgorithm algorithm, Response response)
{
GetHashResult computedHash = GetHash(content, offset, count, algorithm);
AssertResponseHashMatch(computedHash, algorithm, response);
}
/// <summary>
/// Asserts the content of the given stream match the response content hash.
/// </summary>
/// <param name="content">Content to hash.</param>
/// <param name="algorithm">Hash algorithm identifier.</param>
/// <param name="response">Response containing a response hash.</param>
/// <exception cref="ArgumentException">
/// Throws if <paramref name="algorithm"/> is invalid.
/// </exception>
/// <exception cref="InvalidDataException">
/// Throws if the hashes do not match.
/// </exception>
public static void AssertResponseHashMatch(Stream content, TransactionalHashAlgorithm algorithm, Response response)
{
GetHashResult computedHash = GetHash(content, algorithm);
AssertResponseHashMatch(computedHash, algorithm, response);
}
