public void TplDownloadMultipleBlobsTest()
{
MemoryStream blobData1 = new MemoryStream(GetRandomBuffer(2 * 1024));
MemoryStream blobData2 = new MemoryStream(GetRandomBuffer(2 * 1024));
MemoryStream downloaded1 = new MemoryStream();
MemoryStream downloaded2 = new MemoryStream();
CloudPageBlob blob1 = this.testContainer.GetPageBlobReference("blob1");
CloudPageBlob blob2 = this.testContainer.GetPageBlobReference("blob2");
Task[] uploadTasks = new Task[]
{
blob1.UploadFromStreamAsync(blobData1),
blob2.UploadFromStreamAsync(blobData2)
};
Task.WaitAll(uploadTasks);
Task[] downloadTasks = new Task[]
{
blob1.DownloadToStreamAsync(downloaded1),
blob2.DownloadToStreamAsync(downloaded2)
};
Task.WaitAll(downloadTasks);
TestHelper.AssertStreamsAreEqual(blobData1, downloaded1);
TestHelper.AssertStreamsAreEqual(blobData2, downloaded2);
blobData1.Dispose();
blobData2.Dispose();
downloaded1.Dispose();
downloaded2.Dispose();
}
public async Task UseTransactionalCRC64GetTestAsync()
{
BlobRequestOptions optionsWithNoCRC64 = new BlobRequestOptions()
{
ChecksumOptions = new ChecksumOptions {
UseTransactionalCRC64 = false
}
};
BlobRequestOptions optionsWithCRC64 = new BlobRequestOptions()
{
ChecksumOptions = new ChecksumOptions {
UseTransactionalCRC64 = true
}
};
byte[] buffer = GetRandomBuffer(3 * 1024 * 1024);
Crc64Wrapper hasher = new Crc64Wrapper();
hasher.UpdateHash(buffer, 0, buffer.Length);
string crc64 = hasher.ComputeHash();
string lastCheckCRC64 = null;
OperationContext opContextWithCRC64Check = new OperationContext();
opContextWithCRC64Check.ResponseReceived += (_, args) =>
{
if (long.Parse(HttpResponseParsers.GetContentLength(args.Response)) >= buffer.Length)
{
lastCheckCRC64 = HttpResponseParsers.GetContentCRC64(args.Response);
}
};
OperationContext opContextWithCRC64CheckAndInjectedFailure = new OperationContext();
opContextWithCRC64CheckAndInjectedFailure.ResponseReceived += (_, args) =>
{
args.Response.Headers.Remove(Constants.HeaderConstants.ContentCrc64Header);
args.Response.Headers.TryAddWithoutValidation(Constants.HeaderConstants.ContentCrc64Header, "dummy");
if (long.Parse(HttpResponseParsers.GetContentLength(args.Response)) >= buffer.Length)
{
lastCheckCRC64 = HttpResponseParsers.GetContentCRC64(args.Response);
}
};
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
using (Stream blobStream = await blockBlob.OpenWriteAsync())
{
blobStream.Write(buffer, 0, buffer.Length);
blobStream.Write(buffer, 0, buffer.Length);
}
using (Stream stream = new MemoryStream())
{
//lastCheckCRC64 = null;
//blockBlob.DownloadToStream(stream, null, optionsWithNoCRC64, opContextWithCRC64Check);
//Assert.IsNotNull(lastCheckCRC64);
//lastCheckCRC64 = null;
//blockBlob.DownloadToStream(stream, null, optionsWithCRC64, opContextWithCRC64Check);
//Assert.IsNotNull(lastCheckCRC64);
lastCheckCRC64 = "invalid_CRC64";
await blockBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithNoCRC64, opContextWithCRC64Check);
Assert.IsNull(lastCheckCRC64);
lastCheckCRC64 = "invalid_CRC64";
await blockBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithCRC64, opContextWithCRC64Check);
Assert.AreEqual(crc64, lastCheckCRC64);
await TestHelper.ExpectedExceptionAsync <StorageException>(
() => blockBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithCRC64, opContextWithCRC64CheckAndInjectedFailure),
"Calculated CRC64 does not match existing property"
);
lastCheckCRC64 = "invalid_CRC64";
await blockBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoCRC64, opContextWithCRC64Check);
Assert.IsNull(lastCheckCRC64);
StorageException storageEx = await TestHelper.ExpectedExceptionAsync <StorageException>(
() => blockBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithCRC64, opContextWithCRC64Check),
"Downloading more than 4MB with transactional CRC64 should not be supported");
Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException));
lastCheckCRC64 = null;
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithCRC64, opContextWithCRC64Check))
{
blobStream.CopyTo(stream);
Assert.IsNotNull(lastCheckCRC64);
}
lastCheckCRC64 = "invalid_CRC64";
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithNoCRC64, opContextWithCRC64Check))
{
blobStream.CopyTo(stream);
Assert.IsNull(lastCheckCRC64);
}
}
CloudPageBlob pageBlob = container.GetPageBlobReference("blob3");
using (Stream blobStream = await pageBlob.OpenWriteAsync(buffer.Length * 2))
{
blobStream.Write(buffer, 0, buffer.Length);
blobStream.Write(buffer, 0, buffer.Length);
}
using (Stream stream = new MemoryStream())
{
lastCheckCRC64 = "invalid_CRC64";
await pageBlob.DownloadToStreamAsync(stream, null, optionsWithNoCRC64, opContextWithCRC64Check);
Assert.IsNull(lastCheckCRC64);
StorageException storageEx = await TestHelper.ExpectedExceptionAsync <StorageException>(
() => pageBlob.DownloadToStreamAsync(stream, null, optionsWithCRC64, opContextWithCRC64Check),
"Page blob will not have CRC64 set by default; with UseTransactional, download should fail");
lastCheckCRC64 = "invalid_CRC64";
await pageBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithNoCRC64, opContextWithCRC64Check);
Assert.IsNull(lastCheckCRC64);
lastCheckCRC64 = "invalid_CRC64";
await pageBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithCRC64, opContextWithCRC64Check);
Assert.AreEqual(crc64, lastCheckCRC64);
await TestHelper.ExpectedExceptionAsync <StorageException>(
() => pageBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithCRC64, opContextWithCRC64CheckAndInjectedFailure),
"Calculated CRC64 does not match existing property"
);
lastCheckCRC64 = "invalid_CRC64";
await pageBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoCRC64, opContextWithCRC64Check);
Assert.IsNull(lastCheckCRC64);
storageEx = await TestHelper.ExpectedExceptionAsync <StorageException>(
() => pageBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithCRC64, opContextWithCRC64Check),
"Downloading more than 4MB with transactional CRC64 should not be supported");
Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException));
lastCheckCRC64 = null;
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithCRC64, opContextWithCRC64Check))
{
blobStream.CopyTo(stream);
Assert.IsNotNull(lastCheckCRC64);
}
lastCheckCRC64 = "invalid_CRC64";
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithNoCRC64, opContextWithCRC64Check))
{
blobStream.CopyTo(stream);
Assert.IsNull(lastCheckCRC64);
}
}
}
finally
{
await container.DeleteIfExistsAsync();
}
}
public async Task UseTransactionalMD5GetTestAsync()
{
BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions()
{
UseTransactionalMD5 = false,
};
BlobRequestOptions optionsWithMD5 = new BlobRequestOptions()
{
UseTransactionalMD5 = true,
};
byte[] buffer = GetRandomBuffer(3 * 1024 * 1024);
MD5 hasher = MD5.Create();
string md5 = Convert.ToBase64String(hasher.ComputeHash(buffer));
string lastCheckMD5 = null;
int checkCount = 0;
OperationContext opContextWithMD5Check = new OperationContext();
opContextWithMD5Check.ResponseReceived += (_, args) =>
{
if (long.Parse(HttpResponseParsers.GetContentLength(args.Response)) >= buffer.Length)
{
lastCheckMD5 = HttpResponseParsers.GetContentMD5(args.Response);
checkCount++;
}
};
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
using (Stream blobStream = await blockBlob.OpenWriteAsync())
{
blobStream.Write(buffer, 0, buffer.Length);
blobStream.Write(buffer, 0, buffer.Length);
}
checkCount = 0;
using (Stream stream = new MemoryStream())
{
lastCheckMD5 = null;
await blockBlob.DownloadToStreamAsync(stream, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNotNull(lastCheckMD5);
lastCheckMD5 = null;
await blockBlob.DownloadToStreamAsync(stream, null, optionsWithMD5, opContextWithMD5Check);
Assert.IsNotNull(lastCheckMD5);
lastCheckMD5 = "invalid_md5";
await blockBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
lastCheckMD5 = "invalid_md5";
await blockBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithMD5, opContextWithMD5Check);
Assert.AreEqual(md5, lastCheckMD5);
lastCheckMD5 = "invalid_md5";
await blockBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
StorageException storageEx = await TestHelper.ExpectedExceptionAsync <StorageException>(
() => blockBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithMD5, opContextWithMD5Check),
"Downloading more than 4MB with transactional MD5 should not be supported");
Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException));
lastCheckMD5 = null;
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithMD5, opContextWithMD5Check))
{
blobStream.CopyTo(stream);
Assert.IsNotNull(lastCheckMD5);
}
lastCheckMD5 = "invalid_md5";
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithNoMD5, opContextWithMD5Check))
{
blobStream.CopyTo(stream);
Assert.IsNull(lastCheckMD5);
}
}
Assert.AreEqual(9, checkCount);
CloudPageBlob pageBlob = container.GetPageBlobReference("blob3");
using (Stream blobStream = await pageBlob.OpenWriteAsync(buffer.Length * 2))
{
blobStream.Write(buffer, 0, buffer.Length);
blobStream.Write(buffer, 0, buffer.Length);
}
checkCount = 0;
using (Stream stream = new MemoryStream())
{
lastCheckMD5 = "invalid_md5";
await pageBlob.DownloadToStreamAsync(stream, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
StorageException storageEx = await TestHelper.ExpectedExceptionAsync <StorageException>(
() => pageBlob.DownloadToStreamAsync(stream, null, optionsWithMD5, opContextWithMD5Check),
"Page blob will not have MD5 set by default; with UseTransactional, download should fail");
lastCheckMD5 = "invalid_md5";
await pageBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
lastCheckMD5 = "invalid_md5";
await pageBlob.DownloadRangeToStreamAsync(stream, buffer.Length, buffer.Length, null, optionsWithMD5, opContextWithMD5Check);
Assert.AreEqual(md5, lastCheckMD5);
lastCheckMD5 = "invalid_md5";
await pageBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
storageEx = await TestHelper.ExpectedExceptionAsync <StorageException>(
() => pageBlob.DownloadRangeToStreamAsync(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithMD5, opContextWithMD5Check),
"Downloading more than 4MB with transactional MD5 should not be supported");
Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException));
lastCheckMD5 = null;
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithMD5, opContextWithMD5Check))
{
blobStream.CopyTo(stream);
Assert.IsNotNull(lastCheckMD5);
}
lastCheckMD5 = "invalid_md5";
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithNoMD5, opContextWithMD5Check))
{
blobStream.CopyTo(stream);
Assert.IsNull(lastCheckMD5);
}
}
Assert.AreEqual(9, checkCount);
}
finally
{
await container.DeleteIfExistsAsync();
}
}
public async Task DisableContentMD5ValidationTestAsync()
{
byte[] buffer = new byte[1024];
Random random = new Random();
random.NextBytes(buffer);
BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = true,
StoreBlobContentMD5 = true,
};
BlobRequestOptions optionsWithMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = false,
StoreBlobContentMD5 = true,
};
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
using (Stream stream = new NonSeekableMemoryStream(buffer))
{
await blockBlob.UploadFromStreamAsync(stream, null, optionsWithMD5, null);
}
using (Stream stream = new MemoryStream())
{
await blockBlob.DownloadToStreamAsync(stream, null, optionsWithMD5, null);
await blockBlob.DownloadToStreamAsync(stream, null, optionsWithNoMD5, null);
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream;
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream;
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
blockBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
await blockBlob.SetPropertiesAsync();
OperationContext opContext = new OperationContext();
await TestHelper.ExpectedExceptionAsync(
async() => await blockBlob.DownloadToStreamAsync(stream, null, optionsWithMD5, opContext),
opContext,
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
await blockBlob.DownloadToStreamAsync(stream, null, optionsWithNoMD5, null);
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream;
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStreamForRead.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (Stream blobStream = await blockBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream;
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
}
CloudPageBlob pageBlob = container.GetPageBlobReference("blob2");
using (Stream stream = new MemoryStream(buffer))
{
await pageBlob.UploadFromStreamAsync(stream, null, optionsWithMD5, null);
}
using (Stream stream = new MemoryStream())
{
await pageBlob.DownloadToStreamAsync(stream, null, optionsWithMD5, null);
await pageBlob.DownloadToStreamAsync(stream, null, optionsWithNoMD5, null);
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream;
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream;
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
pageBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
await pageBlob.SetPropertiesAsync();
OperationContext opContext = new OperationContext();
await TestHelper.ExpectedExceptionAsync(
async() => await pageBlob.DownloadToStreamAsync(stream, null, optionsWithMD5, opContext),
opContext,
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
await pageBlob.DownloadToStreamAsync(stream, null, optionsWithNoMD5, null);
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream;
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStreamForRead.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (Stream blobStream = await pageBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream;
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
}
}
finally
{
container.DeleteIfExistsAsync().Wait();
}
}
