/// <summary>
/// Test blob reads, expecting lease failure.
/// </summary>
/// <param name="testBlob">The blob to test.</param>
/// <param name="targetBlob">The blob to use for the target of copy operations.</param>
/// <param name="testAccessCondition">The failing access condition to use.</param>
/// <param name="expectedErrorCode">The expected error code.</param>
/// <param name="description">The reason why these calls should fail.</param>
private async Task BlobReadExpectLeaseFailureAsync(CloudBlockBlob testBlob, CloudBlockBlob targetBlob, AccessCondition testAccessCondition, HttpStatusCode expectedStatusCode, string expectedErrorCode, string description)
{
OperationContext operationContext = new OperationContext();
// FetchAttributes is a HEAD request with no extended error info, so it returns with the generic ConditionFailed error code.
await TestHelper.ExpectedExceptionAsync(
async () => await testBlob.FetchAttributesAsync(testAccessCondition, null /* options */, operationContext),
operationContext,
description + "(Fetch Attributes)",
HttpStatusCode.PreconditionFailed);
await TestHelper.ExpectedExceptionAsync(
async () => await testBlob.SnapshotAsync(null /* metadata */, testAccessCondition, null /* options */, operationContext),
operationContext,
description + " (Create Snapshot)",
expectedStatusCode,
expectedErrorCode);
await TestHelper.ExpectedExceptionAsync(
async () => await DownloadTextAsync(testBlob, Encoding.UTF8, testAccessCondition, null /* options */, operationContext),
operationContext,
description + " (Download Text)",
expectedStatusCode,
expectedErrorCode);
await TestHelper.ExpectedExceptionAsync(
async () => await testBlob.OpenReadAsync(testAccessCondition, null /* options */, operationContext),
operationContext,
description + " (Read Stream)",
expectedStatusCode/*,
expectedErrorCode*/);
}
/// <summary>
/// Test blob reads, expecting success.
/// </summary>
/// <param name="testBlob">The blob to test.</param>
/// <param name="targetBlob">The blob to use for the target of copy operations.</param>
/// <param name="testAccessCondition">The access condition to use.</param>
private async Task BlobReadExpectLeaseSuccessAsync(CloudBlockBlob testBlob, AccessCondition testAccessCondition)
{
await testBlob.FetchAttributesAsync(testAccessCondition, null /* options */, null);
await (await testBlob.SnapshotAsync(null /* metadata */, testAccessCondition, null /* options */, null)).DeleteAsync();
await DownloadTextAsync(testBlob, Encoding.UTF8, testAccessCondition, null /* options */, null);
var readStream = await testBlob.OpenReadAsync(testAccessCondition, null /* options */, null);
Stream stream = readStream.AsStreamForRead();
stream.ReadByte();
}
/// <summary>
/// Test blob reads, expecting success.
/// </summary>
/// <param name="testBlob">The blob to test.</param>
/// <param name="testAccessCondition">The access condition to use.</param>
private void BlobReadExpectLeaseSuccessTask(CloudBlockBlob testBlob, AccessCondition testAccessCondition)
{
testBlob.FetchAttributesAsync(testAccessCondition, null /* options */, new OperationContext());
testBlob.SnapshotAsync(null /* metadata */, testAccessCondition, null /* options */, new OperationContext()).Result.Delete();
DownloadTextTask(testBlob, Encoding.UTF8, testAccessCondition, null /* options */);
Stream stream = testBlob.OpenReadAsync(testAccessCondition, null /* options */, new OperationContext()).Result;
stream.ReadByte();
}
public async Task CloudBlobClientMaximumExecutionTimeoutShouldNotBeHonoredForStreamsAsync()
{
CloudBlobClient blobClient = GenerateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(Guid.NewGuid().ToString("N"));
byte[] buffer = BlobTestBase.GetRandomBuffer(1024 * 1024);
try
{
await container.CreateAsync();
blobClient.DefaultRequestOptions.MaximumExecutionTime = TimeSpan.FromSeconds(30);
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
CloudPageBlob pageBlob = container.GetPageBlobReference("blob2");
blockBlob.StreamWriteSizeInBytes = 1024 * 1024;
blockBlob.StreamMinimumReadSizeInBytes = 1024 * 1024;
pageBlob.StreamWriteSizeInBytes = 1024 * 1024;
pageBlob.StreamMinimumReadSizeInBytes = 1024 * 1024;
using (var bos = await blockBlob.OpenWriteAsync())
{
DateTime start = DateTime.Now;
for (int i = 0; i < 7; i++)
{
await bos.WriteAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last write
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
await bos.WriteAsync(buffer, 0, buffer.Length);
await bos.CommitAsync();
}
using (Stream bis = (await blockBlob.OpenReadAsync()))
{
DateTime start = DateTime.Now;
int total = 0;
while (total < 7 * 1024 * 1024)
{
total += await bis.ReadAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last read
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
while (true)
{
int count = await bis.ReadAsync(buffer, 0, buffer.Length);
total += count;
if (count == 0)
{
break;
}
}
}
using (var bos = await pageBlob.OpenWriteAsync(8 * 1024 * 1024))
{
DateTime start = DateTime.Now;
for (int i = 0; i < 7; i++)
{
await bos.WriteAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last write
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
await bos.WriteAsync(buffer, 0, buffer.Length);
await bos.CommitAsync();
}
using (Stream bis = (await pageBlob.OpenReadAsync()))
{
DateTime start = DateTime.Now;
int total = 0;
while (total < 7 * 1024 * 1024)
{
total += await bis.ReadAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last read
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
while (true)
{
int count = await bis.ReadAsync(buffer, 0, buffer.Length);
total += count;
if (count == 0)
{
break;
}
}
}
}
finally
{
blobClient.DefaultRequestOptions.MaximumExecutionTime = null;
container.DeleteIfExistsAsync().Wait();
}
}
private async Task<bool> TryResumeFromState(CloudBlockBlob stateBlob)
{
using (var stream = await stateBlob.OpenReadAsync())
using (var reader = new JsonTextReader(new StreamReader(stream)))
{
var jsonSerializer = JsonSerializer.CreateDefault();
this.state = jsonSerializer.Deserialize<OnlineTrainerState>(reader);
}
this.telemetry.TrackTrace(
$"Resume from '{stateBlob.Uri}'",
SeverityLevel.Verbose,
this.state.PartitionsDetailed);
return await this.TryLoadModel();
}
public async Task DisableContentMD5ValidationTestAsync()
{
byte[] buffer = new byte[1024];
Random random = new Random();
random.NextBytes(buffer);
BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = true,
StoreBlobContentMD5 = true,
};
BlobRequestOptions optionsWithMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = false,
StoreBlobContentMD5 = true,
};
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
using (Stream stream = new NonSeekableMemoryStream(buffer))
{
await blockBlob.UploadFromStreamAsync(stream.AsInputStream(), null, optionsWithMD5, null);
}
using (Stream stream = new MemoryStream())
{
await blockBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithMD5, null);
await blockBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithNoMD5, null);
using (var blobStream = await blockBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
using (var blobStream = await blockBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
blockBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
await blockBlob.SetPropertiesAsync();
OperationContext opContext = new OperationContext();
await TestHelper.ExpectedExceptionAsync(
async() => await blockBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithMD5, opContext),
opContext,
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
await blockBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithNoMD5, null);
using (var blobStream = await blockBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStreamForRead.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (var blobStream = await blockBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
}
CloudPageBlob pageBlob = container.GetPageBlobReference("blob2");
using (Stream stream = new MemoryStream(buffer))
{
await pageBlob.UploadFromStreamAsync(stream.AsInputStream(), null, optionsWithMD5, null);
}
using (Stream stream = new MemoryStream())
{
await pageBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithMD5, null);
await pageBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithNoMD5, null);
using (var blobStream = await pageBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
using (var blobStream = await pageBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
pageBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
await pageBlob.SetPropertiesAsync();
OperationContext opContext = new OperationContext();
await TestHelper.ExpectedExceptionAsync(
async() => await pageBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithMD5, opContext),
opContext,
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
await pageBlob.DownloadToStreamAsync(stream.AsOutputStream(), null, optionsWithNoMD5, null);
using (var blobStream = await pageBlob.OpenReadAsync(null, optionsWithMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStreamForRead.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (var blobStream = await pageBlob.OpenReadAsync(null, optionsWithNoMD5, null))
{
Stream blobStreamForRead = blobStream.AsStreamForRead();
int read;
do
{
read = await blobStreamForRead.ReadAsync(buffer, 0, buffer.Length);
}while (read > 0);
}
}
}
finally
{
container.DeleteIfExistsAsync().AsTask().Wait();
}
}
public async Task BlockBlobReadLockToETagTestAsync()
{
byte[] outBuffer = new byte[1 * 1024 * 1024];
byte[] buffer = GetRandomBuffer(2 * outBuffer.Length);
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
blob.StreamMinimumReadSizeInBytes = outBuffer.Length;
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
await blob.UploadFromStreamAsync(wholeBlob);
}
OperationContext opContext = new OperationContext();
using (Stream blobStream = await blob.OpenReadAsync(null, null, opContext))
{
Stream blobStreamForRead = blobStream;
await blobStreamForRead.ReadAsync(outBuffer, 0, outBuffer.Length);
await blob.SetMetadataAsync();
await TestHelper.ExpectedExceptionAsync(
async() => await blobStreamForRead.ReadAsync(outBuffer, 0, outBuffer.Length),
opContext,
"Blob read stream should fail if blob is modified during read",
HttpStatusCode.PreconditionFailed);
}
opContext = new OperationContext();
using (Stream blobStream = await blob.OpenReadAsync(null, null, opContext))
{
Stream blobStreamForRead = blobStream;
long length = blobStreamForRead.Length;
await blob.SetMetadataAsync();
await TestHelper.ExpectedExceptionAsync(
async() => await blobStreamForRead.ReadAsync(outBuffer, 0, outBuffer.Length),
opContext,
"Blob read stream should fail if blob is modified during read",
HttpStatusCode.PreconditionFailed);
}
opContext = new OperationContext();
AccessCondition accessCondition = AccessCondition.GenerateIfNotModifiedSinceCondition(DateTimeOffset.Now.Subtract(TimeSpan.FromHours(1)));
await blob.SetMetadataAsync();
await TestHelper.ExpectedExceptionAsync(
async() => await blob.OpenReadAsync(accessCondition, null, opContext),
opContext,
"Blob read stream should fail if blob is modified during read",
HttpStatusCode.PreconditionFailed);
}
finally
{
container.DeleteIfExistsAsync().Wait();
}
}