protected override async Task AppendInternal(bool async, CancellationToken cancellationToken)
{
if (_buffer.Length > 0)
{
_buffer.Position = 0;
HttpRange httpRange = new HttpRange(_writeIndex, _buffer.Length);
await _fileClient.UploadRangeInternal(
range : httpRange,
content : _buffer,
// TODO #27253
//options: new ShareFileUploadRangeOptions
//{
// //TransactionalHashingOptions = _hashingOptions,
// ProgressHandler = _progressHandler,
// Conditions = _conditions
//},
rangeContentMD5 : default,
public async Task <Stream> DecryptInternal(
Stream content,
Metadata metadata,
HttpRange originalRange,
string receivedContentRange,
bool async,
CancellationToken cancellationToken)
{
ContentRange?contentRange = string.IsNullOrWhiteSpace(receivedContentRange)
? default
: ContentRange.Parse(receivedContentRange);
EncryptionData encryptionData = GetAndValidateEncryptionDataOrDefault(metadata);
if (encryptionData == default)
{
return(await TrimStreamInternal(content, originalRange, contentRange, alreadyTrimmedOffsetAmount : 0, async, cancellationToken).ConfigureAwait(false));
}
bool ivInStream = originalRange.Offset >= Constants.ClientSideEncryption.EncryptionBlockSize;
// this method throws when key cannot be resolved. Blobs is intended to throw on this failure.
var plaintext = await _decryptor.DecryptReadInternal(
content,
encryptionData,
ivInStream,
CanIgnorePadding(contentRange),
async,
cancellationToken).ConfigureAwait(false);
int v2StartRegion0Indexed = (int)((contentRange?.Start / encryptionData.EncryptedRegionInfo?.GetTotalRegionLength()) ?? 0);
int alreadyTrimmedOffset = encryptionData.EncryptionAgent.EncryptionVersion switch
{
ClientSideEncryptionVersion.V1_0 => ivInStream ? Constants.ClientSideEncryption.EncryptionBlockSize : 0,
// first block is special case where we don't want to communicate a trim. Otherwise communicate nonce length * 1-indexed start region + tag length * 0-indexed region
ClientSideEncryptionVersion.V2_0 => contentRange?.Start > 0
? (-encryptionData.EncryptedRegionInfo.NonceLength * (v2StartRegion0Indexed)) - (Constants.ClientSideEncryption.V2.TagSize * v2StartRegion0Indexed)
: 0,
_ => throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported()
};
return(await TrimStreamInternal(plaintext, originalRange, contentRange, alreadyTrimmedOffset, async, cancellationToken).ConfigureAwait(false));
}
public void Equality()
{
var nullRange = new HttpRange(0, null);
var nullStart = new HttpRange(0, 5);
var nullEnd = new HttpRange(5, null);
var r5_10 = new HttpRange(5, 10);
var r5_10_copy = new HttpRange(5, 10);
Assert.AreEqual(r5_10, r5_10_copy);
Assert.IsTrue(r5_10 == r5_10_copy);
Assert.IsFalse(r5_10 == nullRange);
Assert.IsFalse(r5_10 == nullStart);
Assert.IsFalse(r5_10 == nullEnd);
Assert.IsFalse(r5_10 != r5_10_copy);
Assert.IsTrue(r5_10 != nullRange);
Assert.IsTrue(r5_10 != nullStart);
Assert.IsTrue(r5_10 != nullEnd);
}
public static void AppendToAzureFile(ShareFileClient azureFile, byte[] buffer)
{
int bufferLength = buffer.Length;
azureFile.Create(bufferLength);
int maxChunkSize = 4 * 1024;
for (int offset = 0; offset < bufferLength; offset += maxChunkSize)
{
int chunkSize = Math.Min(maxChunkSize, bufferLength - offset);
MemoryStream chunk = new MemoryStream();
chunk.Write(buffer, offset, chunkSize);
chunk.Position = 0;
HttpRange httpRange = new HttpRange(offset, chunkSize);
var resp = azureFile.UploadRange(httpRange, chunk);
}
}
/// <inheritdoc/>
public async Task <BlobDownloadInfo> DownloadHttpRangeAsync(Uri blobUri, HttpRange httpRange = default)
{
_ = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
var blobBaseClient = new BlobBaseClient(blobUri, _tokenCredential);
// Note: when the httpRange struct is omitted it defaults to 'default' which downloads the entire blob.
BlobDownloadInfo blobDownloadInfo;
try
{
blobDownloadInfo = (await blobBaseClient.DownloadAsync(httpRange).ConfigureAwait(false)).Value;
}
catch (Exception e)
{
var message = $"Could not download the HTTP range for {blobUri}.";
_log.LogError(message, e);
throw new Exception(message, e);
}
return(blobDownloadInfo);
}
public async Task <Response> DownloadToAsync(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}");
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Task <Response <BlobDownloadStreamingResult> > initialResponseTask =
_client.DownloadStreamingAsync(
new BlobDownloadOptions
{
Range = initialRange,
Conditions = conditions,
TransactionalHashingOptions = _hashingOptions,
ProgressHandler = _progress,
},
cancellationToken);
Response <BlobDownloadStreamingResult> initialResponse = null;
try
{
initialResponse = await initialResponseTask.ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.InvalidRange)
{
initialResponse = await _client.DownloadStreamingAsync(
new BlobDownloadOptions
{
Range = default,
public override byte ReadByte()
{
if (_blobLength == 0)
{
return(0);
}
if (_currPage == null || _prevPageOffset != GetCurrentPageOffset())
{
// download and cache the current page.
_currPage ??= new byte[512];
var range = new HttpRange(_currPageOffset, 512);
BlobDownloadInfo download = _pageBlobClient.Download(range);
download.Content.Read(_currPage, 0, 512);
}
var pos = _currPagePosition - _currPageOffset;
var b = _currPage[pos];
_currPagePosition += 1;
return(b);
}
internal async Task <Response> DownloadToAsync(
Stream destination,
Uri endpoint,
CancellationToken cancellationToken)
{
var initialRange = new HttpRange(0, _initialRangeSize);
Task <Response <Stream> > initialResponseTask =
_client.DownloadStreamingAsync(
endpoint,
initialRange,
cancellationToken);
Response <Stream> initialResponse;
try
{
initialResponse = await initialResponseTask.ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.Status == 416) //Invalid Range
{
initialResponseTask = _client.DownloadStreamingAsync(
endpoint,
range: default,
public async Task <Response> DownloadToAsync(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientConfiguration.ClientDiagnostics.CreateScope(_operationName);
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Task <Response <BlobDownloadStreamingResult> > initialResponseTask =
_client.DownloadStreamingInternal(
initialRange,
conditions,
rangeGetContentHash: default,
private static async Task <Stream> TrimStreamInternal(
Stream stream,
HttpRange originalRange,
ContentRange?receivedRange,
bool pulledOutIV,
bool async,
CancellationToken cancellationToken)
{
// retrim start of stream to original requested location
// keeping in mind whether we already pulled the IV out of the stream as well
int gap = (int)(originalRange.Offset - (receivedRange?.Start ?? 0))
- (pulledOutIV ? Constants.ClientSideEncryption.EncryptionBlockSize : 0);
int read = 0;
while (gap > read)
{
int toRead = gap - read;
// throw away initial bytes we want to trim off; stream cannot seek into future
if (async)
{
read += await stream.ReadAsync(new byte[toRead], 0, toRead, cancellationToken).ConfigureAwait(false);
}
else
{
read += stream.Read(new byte[toRead], 0, toRead);
}
}
if (originalRange.Length.HasValue)
{
stream = new WindowStream(stream, originalRange.Length.Value);
}
return(stream);
}
/// <summary>
/// The <see cref="StageBlockFromUriAsync"/> operation creates a new
/// block to be committed as part of a blob where the contents are
/// read from the <paramref name="sourceUri" />.
///
/// For more information, see <see href="https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url"/>.
/// </summary>
/// <param name="sourceUri">
/// Specifies the <see cref="Uri"/> of the source blob. The value may
/// be a URL of up to 2 KB in length that specifies a blob. The
/// source blob must either be public or must be authenticated via a
/// shared access signature. If the source blob is public, no
/// authentication is required to perform the operation.
/// </param>
/// <param name="base64BlockID">
/// A valid Base64 string value that identifies the block. Prior to
/// encoding, the string must be less than or equal to 64 bytes in
/// size. For a given blob, the length of the value specified for
/// the <paramref name="base64BlockID"/> parameter must be the same
/// size for each block. Note that the Base64 string must be
/// URL-encoded.
/// </param>
/// <param name="sourceRange">
/// Optionally uploads only the bytes of the blob in the
/// <paramref name="sourceUri"/> in the specified range. If this is
/// not specified, the entire source blob contents are uploaded as a
/// single block.
/// </param>
/// <param name="sourceContentHash">
/// Optional MD5 hash of the block content from the
/// <paramref name="sourceUri"/>. This hash is used to verify the
/// integrity of the block during transport of the data from the Uri.
/// When this hash is specified, the storage service compares the hash
/// of the content that has arrived from the <paramref name="sourceUri"/>
/// with this value. Note that this md5 hash is not stored with the
/// blob. If the two hashes do not match, the operation will fail
/// with a <see cref="StorageRequestFailedException"/>.
/// </param>
/// <param name="sourceAccessConditions">
/// Optional <see cref="HttpAccessConditions"/> to add
/// conditions on the copying of data from this source blob.
/// </param>
/// <param name="leaseAccessConditions">
/// Optional <see cref="LeaseAccessConditions"/> to add
/// conditions on the staging of this block.
/// </param>
/// <param name="cancellation">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Task{Response{BlobContentInfo}}"/> describing the
/// state of the updated block blob.
/// </returns>
/// <remarks>
/// A <see cref="StorageRequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
public async Task <Response <BlobContentInfo> > StageBlockFromUriAsync(
Uri sourceUri,
string base64BlockID,
HttpRange sourceRange = default,
byte[] sourceContentHash = default,
HttpAccessConditions?sourceAccessConditions = default,
LeaseAccessConditions?leaseAccessConditions = default,
CancellationToken cancellation = default)
{
using (this.Pipeline.BeginLoggingScope(nameof(BlockBlobClient)))
{
this.Pipeline.LogMethodEnter(
nameof(BlockBlobClient),
message:
$"{nameof(this.Uri)}: {this.Uri}\n" +
$"{nameof(base64BlockID)}: {base64BlockID}\n" +
$"{nameof(sourceUri)}: {sourceUri}\n" +
$"{nameof(leaseAccessConditions)}: {leaseAccessConditions}");
try
{
return(await BlobRestClient.BlockBlob.StageBlockFromUriAsync(
this.Pipeline,
this.Uri,
contentLength : default,
private static async Task <Stream> TrimStreamInternal(
Stream stream,
HttpRange originalRange,
ContentRange?receivedRange,
// iv or nonce in stream could have already been trimmed during decryption
int alreadyTrimmedOffsetAmount,
bool async,
CancellationToken cancellationToken)
{
// retrim start of stream to original requested location
// keeping in mind whether we already trimmed due to an IV or nonce
int gap = (int)(originalRange.Offset - (receivedRange?.Start ?? 0)) - alreadyTrimmedOffsetAmount;
int read = 0;
while (gap > read)
{
int toRead = gap - read;
// throw away initial bytes we want to trim off; stream cannot seek into future
if (async)
{
read += await stream.ReadAsync(new byte[toRead], 0, toRead, cancellationToken).ConfigureAwait(false);
}
else
{
read += stream.Read(new byte[toRead], 0, toRead);
}
}
if (originalRange.Length.HasValue)
{
stream = WindowStream.GetWindow(stream, originalRange.Length.Value);
}
return(stream);
}
/// <inheritdoc/>
public async Task <CachedHttpRangeContent> GetOrDownloadContentAsync(Uri blobUri, long desiredOffset, long desiredSize, StorageClientProviderContext context)
{
_ = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
_ = context ?? throw new ArgumentNullException(nameof(context));
// fixup for default size.
if (desiredSize == UseDefaultLength)
{
desiredSize = DefaultLength;
}
if (desiredOffset < 0)
{
throw new ArgumentOutOfRangeException(nameof(desiredOffset), $"Must be greater than zero. {desiredOffset}");
}
if (desiredSize < 0)
{
throw new ArgumentOutOfRangeException(nameof(desiredSize), $"Must be greater than zero. {desiredSize}");
}
if (desiredSize > MaxCachedBytes)
{
throw new ArgumentOutOfRangeException(nameof(desiredSize), $"Must be less than or equal to {MaxCachedBytes}.");
}
// Since the cachedContentTree is only managed on a single-URI basis, we need
// to determine if the cache content is for the requested URI. If not, flush
// out the cache as we're starting over for a new URI.
string uriString = blobUri.ToString();
if (uriString != lastUriCached)
{
CleanUpCachedContentTree();
lastUriCached = uriString;
}
else
{
// It's for the same URI as last call, so check cache for ranges that contain this offset.
var cachedHttpRangeContentEntry = cachedContentTree
.Query((int)desiredOffset)
.OrderByDescending(e => e.CachedHttpRange.Offset)
.FirstOrDefault();
if (cachedHttpRangeContentEntry != default(CachedHttpRangeContent))
{
// Console.WriteLine($"Found Range:\t\t {cachedHttpRangeContentEntry.Range.Offset},\t\t {cachedHttpRangeContentEntry.Range.Offset + cachedHttpRangeContentEntry.Range.Length - 1}");
_log.LogEventObject(LogEventIds.FoundCachedHttpRange, new { httpRange = cachedHttpRangeContentEntry.CachedHttpRange, desiredOffset });
return(cachedHttpRangeContentEntry);
}
}
// No luck, nothing suitable in the cache so we're going to have to download a new range to cover
// the request. Clean out the whole tree if we are about to exceed the MaxMemorySize.
if (totalContentLength + desiredSize >= MaxCachedBytes)
{
CleanUpCachedContentTree();
}
int downloadedContentLength = 0;
MemoryStream memStream = null;
var requestedHttpRange = new HttpRange(desiredOffset, desiredSize);
try
{
using var downloadResponse = await DownloadHttpRangeAsync(blobUri, context, requestedHttpRange).ConfigureAwait(false);
downloadedContentLength = (int)downloadResponse.ContentLength;
totalContentLength += downloadedContentLength;
#pragma warning disable CA2000 // Dispose objects before losing scope
memStream = new MemoryStream(downloadedContentLength);
#pragma warning restore CA2000 // Dispose objects before losing scope
downloadResponse.Content.CopyTo(memStream);
}
catch (Exception e) when(
e is ArgumentOutOfRangeException ||
e is ArgumentNullException ||
e is NotSupportedException ||
e is ObjectDisposedException ||
e is IOException)
{
_log.LogExceptionObject(LogEventIds.FailedToDownloadContentInStorageService, e, new { blobUri, httpRange = requestedHttpRange });
throw new GridwichStorageServiceException(blobUri, "Could not download content for a blob.",
LogEventIds.FailedToDownloadContentInStorageService, context.ClientRequestIdAsJObject, e);
}
var actualHttpRange = new HttpRange(desiredOffset, downloadedContentLength);
var cachedHttpRangeContent = new CachedHttpRangeContent(actualHttpRange, memStream);
cachedContentTree.Add((int)actualHttpRange.Offset, (int)(actualHttpRange.Offset + actualHttpRange.Length - 1), cachedHttpRangeContent);
// Console.WriteLine($"Added Range:\t\t {actualHttpRange.Offset},\t\t {actualHttpRange.Offset + actualHttpRange.Length - 1}");
_log.LogEventObject(LogEventIds.HttpRangeDownloadedFinished, new { httpRange = actualHttpRange, desiredOffset });
return(cachedHttpRangeContent);
}
/// <summary>
///
/// <para>DownloadFile:</para>
///
/// <para>Downloads a file from File Service and stores locally/or to stream, caller thread will be blocked before it is done</para>
///
/// <para>Check <seealso cref="IBFileServiceInterface.DownloadFile"/> for detailed documentation</para>
///
/// </summary>
public bool DownloadFile(string _BucketName, string _KeyInBucket, BStringOrStream _Destination, Action <string> _ErrorMessageAction = null, ulong _StartIndex = 0, ulong _Size = 0)
{
BlobContainerClient ContainerClient = AServiceClient.GetBlobContainerClient(_BucketName);
BlobClient Blob = ContainerClient.GetBlobClient(_KeyInBucket);
Response <BlobProperties> Response = Blob.GetProperties();
if (AServiceClient == null)
{
_ErrorMessageAction?.Invoke("BFileServiceAZ->DownloadFile: AServiceClient is null.");
return(false);
}
if (!CheckFileExistence(
_BucketName,
_KeyInBucket,
out bool bExists,
_ErrorMessageAction
))
{
_ErrorMessageAction?.Invoke("BFileServiceAZ->DownloadFile: CheckFileExistence failed.");
return(false);
}
if (!bExists)
{
_ErrorMessageAction?.Invoke("BFileServiceAZ->DownloadFile: File does not exist in the File Service.");
return(false);
}
HttpRange Range = default(HttpRange);
if (_Size > 0)
{
Range = new HttpRange((long)_StartIndex, (long)(_StartIndex + _Size));
}
try
{
if (_Destination.Type == EBStringOrStreamEnum.String)
{
using (FileStream FS = File.Create(_Destination.String))
{
BlobDownloadInfo DlInfo = Blob.Download(Range).Value;
DlInfo.Content.CopyTo(FS);
DlInfo.Dispose();
}
if (!BUtility.DoesFileExist(
_Destination.String,
out bool bLocalFileExists,
_ErrorMessageAction))
{
_ErrorMessageAction?.Invoke("BFileServiceAZ->DownloadFile: DoesFileExist failed.");
return(false);
}
if (!bLocalFileExists)
{
_ErrorMessageAction?.Invoke("BFileServiceAZ->DownloadFile: Download finished, but still file does not locally exist.");
return(false);
}
}
else
{
if (_Destination.Stream == null)
{
_ErrorMessageAction?.Invoke("BFileServiceAZ->DownloadFile: Destination stream is null.");
return(false);
}
BlobDownloadInfo DlInfo = Blob.Download(Range).Value;
DlInfo.Content.CopyTo(_Destination.Stream);
DlInfo.Dispose();
try
{
_Destination.Stream.Position = 0;
}
catch (Exception) { }
}
}
catch (Exception e)
{
_ErrorMessageAction?.Invoke("BFileServiceAZ->DownloadFile: " + e.Message + ", Trace: " + e.StackTrace);
return(false);
}
return(true);
}
protected override async Task <Response> DownloadPartitionAsync(ShareFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, HttpRange range = default)
{
AssertSupportsHashAlgorithm(hashingOptions?.Algorithm ?? default);
var response = await client.DownloadAsync(new ShareFileDownloadOptions
{
TransactionalHashingOptions = hashingOptions,
Range = range
});
await response.Value.Content.CopyToAsync(destination);
return(response.GetRawResponse());
}
/// <summary>
/// Given <paramref name="ranges"/>, download content and write it to <paramref name="destinationStream"/>.
/// </summary>
/// <typeparam name="P">
/// Response type when downloading a single partition.
/// </typeparam>
/// <param name="destinationStream">
/// The stream to write content into.
/// </param>
/// <param name="etag">
/// The ETag of the content, for concurrency detection.
/// </param>
/// <param name="ranges">
/// The ordered set of ranges to download.
/// </param>
/// <param name="downloadPartitionAsync">
/// Returns a Task that will download a single partition of a stream (given the
/// partition's stream, sequence number, whether to execute it
/// async, and a cancellation token).
/// </param>
/// <param name="writePartitionAsync">
/// Returns a Task that writes the content stream into the destination stream (given the
/// download response return by <paramref name="downloadPartitionAsync"/> and
/// <paramref name="destinationStream"/>).
/// </param>
/// <param name="maximumActivePartitionCount">
/// The maximum number of partitions to download in parallel.
/// </param>
/// <param name="maximumLoadedPartitionCount">
/// The maximum number of partitions to retain in memory.
/// </param>
/// <param name="async">
/// Whether to perform the download asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns></returns>
/// <remarks>
/// This method assumes that individual downloads are automatically retried.
/// </remarks>
private static async Task DownloadRangesImplAsync <P>(
Stream destinationStream,
ETag etag,
IEnumerable <HttpRange> ranges,
Func <ETag, HttpRange, bool, CancellationToken, Task <Response <P> > > downloadPartitionAsync,
Func <Response <P>, Stream, bool, CancellationToken, Task> writePartitionAsync,
int maximumActivePartitionCount,
int maximumLoadedPartitionCount,
bool async,
CancellationToken cancellationToken)
{
// Use a queue to accumulate download tasks and return them in FIFO order.
// Not using a ConcurrentQueue, since we aren't going to write using multiple threads here.
// Based on prior research, we are better off just downloading the ranges, and writing them in order:
// - Required for a non-seekable destination stream;
// - Better performance than a MemoryMappedViewStream, because the file system won't have to zero out
// spans skipped during random writes;
// - Not necessarily as performant as an in-memory seekable stream, but memory streams probably aren't in the
// size range where parallel download is really going to be useful anyway.
//
// We will still download in parallel, but limit ourselves to a maximum number of responses retained in memory,
// and only await the head of the queue.
var activeTaskQueue = new Queue <Task <Response <P> > >();
var loadedResponseQueue = new Queue <Response <P> >();
IEnumerator <HttpRange> rangesEnumerator = ranges.GetEnumerator();
while (true)
{
// Keep the queues filled. We could be more interesting with background threads and various semaphores or locks,
// but this should be good-enough for the download case, given the ordering restriction.
while (activeTaskQueue.Any() && activeTaskQueue.Peek().Status != TaskStatus.Running)
{
cancellationToken.ThrowIfCancellationRequested();
Task <Response <P> > responseTask = activeTaskQueue.Dequeue();
Response <P> response =
async
? await responseTask.ConfigureAwait(false)
: responseTask.EnsureCompleted();
loadedResponseQueue.Enqueue(response);
}
while (
activeTaskQueue.Count < maximumActivePartitionCount &&
(activeTaskQueue.Count + loadedResponseQueue.Count < maximumLoadedPartitionCount)
)
{
if (!rangesEnumerator.MoveNext())
{
break;
}
HttpRange currentRange = rangesEnumerator.Current;
cancellationToken.ThrowIfCancellationRequested();
Task <Task <Response <P> > > newTask = Task.Factory.StartNew(
async() => await downloadPartitionAsync(etag, currentRange, async, cancellationToken).ConfigureAwait(false),
cancellationToken,
TaskCreationOptions.None,
TaskScheduler.Default
);
activeTaskQueue.Enqueue(newTask.Unwrap());
}
if (loadedResponseQueue.Any())
{
// await the completion of the head task, then write it to the destination
cancellationToken.ThrowIfCancellationRequested();
Response <P> response = loadedResponseQueue.Dequeue();
Task writePartitionTask = writePartitionAsync(response, destinationStream, async, cancellationToken);
if (async)
{
await writePartitionTask.ConfigureAwait(false);
}
else
{
writePartitionTask.EnsureCompleted();
}
response.GetRawResponse().Dispose();
}
else if (!activeTaskQueue.Any())
{
// all downloads are completed
break;
}
}
}
protected override async Task <Response> DownloadPartitionAsync(DataLakeFileClient client, Stream destination, DownloadTransactionalHashingOptions hashingOptions, HttpRange range = default)
{
var response = await client.ReadAsync(new DataLakeFileReadOptions
{
TransactionalHashingOptions = hashingOptions,
Range = range
});
await response.Value.Content.CopyToAsync(destination);
return(response.GetRawResponse());
}
/// <summary>
/// Initializes a new instance of the <see cref="CachedHttpRangeContent"/> class.
/// </summary>
/// <param name="range">range.</param>
/// <param name="memoryStream">memoryStream.</param>
public CachedHttpRangeContent(HttpRange range, MemoryStream memoryStream)
{
CachedHttpRange = range;
CachedMemoryStream = memoryStream;
}
/// <summary>
/// Test some of the file storage operations.
/// </summary>
public async Task RunFileStorageOperationsAsync()
{
// These are used in the finally block to clean up the objects created during the demo.
ShareClient shareClient = null;
ShareFileClient shareFileClient = null;
ShareDirectoryClient fileDirectory = null;
BlobClient targetBlob = null;
BlobContainerClient blobContainer = null;
string destFile = null;
string downloadFolder = null;
// Name to be used for the file when downloading it so you can inspect it locally
string downloadFile = null;
try
{
//***** Setup *****//
Console.WriteLine("Getting reference to the storage account.");
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
string storageConnectionString = ConfigurationManager.AppSettings.Get("StorageConnectionString");
string storageAccountName = ConfigurationManager.AppSettings.Get("StorageAccountName");
string storageAccountKey = ConfigurationManager.AppSettings.Get("StorageAccountKey");
Console.WriteLine("Instantiating file client.");
// Create a share client for interacting with the file service.
var shareServiceClient = new ShareServiceClient(storageConnectionString);
// Create the share name -- use a guid in the name so it's unique.
// This will also be used as the container name for blob storage when copying the file to blob storage.
string shareName = "demotest-" + System.Guid.NewGuid().ToString();
// Name of folder to put the files in
string sourceFolder = "testfolder";
// Name of file to upload and download
string testFile = "HelloWorld.png";
// Folder where the HelloWorld.png file resides
string localFolder = @".\";
// It won't let you download in the same folder as the exe file,
// so use a temporary folder with the same name as the share.
downloadFolder = Path.Combine(Path.GetTempPath(), shareName);
//***** Create a file share *****//
// Create the share if it doesn't already exist.
Console.WriteLine("Creating share with name {0}", shareName);
shareClient = shareServiceClient.GetShareClient(shareName);
try
{
await shareClient.CreateIfNotExistsAsync();
Console.WriteLine(" Share created successfully.");
}
catch (RequestFailedException exRequest)
{
Common.WriteException(exRequest);
Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample.");
Console.WriteLine("Press any key to exit");
Console.ReadLine();
throw;
}
catch (Exception ex)
{
Console.WriteLine(" Exception thrown creating share.");
Common.WriteException(ex);
throw;
}
//***** Create a directory on the file share *****//
// Create a directory on the share.
Console.WriteLine("Creating directory named {0}", sourceFolder);
ShareDirectoryClient rootDirectory = shareClient.GetRootDirectoryClient();
// If the source folder is null, then use the root folder.
// If the source folder is specified, then get a reference to it.
if (string.IsNullOrWhiteSpace(sourceFolder))
{
// There is no folder specified, so return a reference to the root directory.
fileDirectory = rootDirectory;
Console.WriteLine(" Using root directory.");
}
else
{
// There was a folder specified, so return a reference to that folder.
fileDirectory = rootDirectory.GetSubdirectoryClient(sourceFolder);
await fileDirectory.CreateIfNotExistsAsync();
Console.WriteLine(" Directory created successfully.");
}
//***** Upload a file to the file share *****//
// Get a file client.
shareFileClient = fileDirectory.GetFileClient(testFile);
// Upload a file to the share.
Console.WriteLine("Uploading file {0} to share", testFile);
// Set up the name and path of the local file.
string sourceFile = Path.Combine(localFolder, testFile);
if (File.Exists(sourceFile))
{
using (FileStream stream = File.OpenRead(sourceFile))
{
// Upload from the local file to the file share in azure.
await shareFileClient.CreateAsync(stream.Length);
await shareFileClient.UploadAsync(stream);
}
Console.WriteLine(" Successfully uploaded file to share.");
}
else
{
Console.WriteLine("File not found, so not uploaded.");
}
//***** Get list of all files/directories on the file share*****//
// List all files/directories under the root directory.
Console.WriteLine("Getting list of all files/directories under the root directory of the share.");
var fileList = rootDirectory.GetFilesAndDirectoriesAsync();
// Print all files/directories listed above.
await foreach (ShareFileItem listItem in fileList)
{
// listItem type will be ShareClient or ShareDirectoryClient.
Console.WriteLine(" - {0} (type: {1})", listItem.Name, listItem.GetType());
}
Console.WriteLine("Getting list of all files/directories in the file directory on the share.");
// Now get the list of all files/directories in your directory.
// Ordinarily, you'd write something recursive to do this for all directories and subdirectories.
fileList = fileDirectory.GetFilesAndDirectoriesAsync();
// Print all files/directories in the folder.
await foreach (ShareFileItem listItem in fileList)
{
// listItem type will be a file or directory
Console.WriteLine(" - {0} (IsDirectory: {1})", listItem.Name, listItem.IsDirectory);
}
//***** Download a file from the file share *****//
// Download the file to the downloadFolder in the temp directory.
// Check and if the directory doesn't exist (which it shouldn't), create it.
Console.WriteLine("Downloading file from share to local temp folder {0}.", downloadFolder);
if (!Directory.Exists(downloadFolder))
{
Directory.CreateDirectory(downloadFolder);
}
// Download the file.
ShareFileDownloadInfo download = await shareFileClient.DownloadAsync();
downloadFile = Path.Combine(downloadFolder, testFile);
using (FileStream stream = File.OpenWrite(downloadFile))
{
await download.Content.CopyToAsync(stream);
}
Console.WriteLine(" Successfully downloaded file from share to local temp folder.");
//***** Copy a file from the file share to blob storage, then abort the copy *****//
// Copies can sometimes complete before there's a chance to abort.
// If that happens with the file you're testing with, try copying the file
// to a storage account in a different region. If it still finishes too fast,
// try using a bigger file and copying it to a different region. That will almost always
// take long enough to give you time to abort the copy.
// If you want to change the file you're testing the Copy with without changing the value for the
// rest of the sample code, upload the file to the share, then assign the name of the file
// to the testFile variable right here before calling GetFileClient.
// Then it will use the new file for the copy and abort but the rest of the code
// will still use the original file.
ShareFileClient shareFileCopy = fileDirectory.GetFileClient(testFile);
// Upload a file to the share.
Console.WriteLine("Uploading file {0} to share", testFile);
// Set up the name and path of the local file.
string sourceFileCopy = Path.Combine(localFolder, testFile);
using (FileStream stream = File.OpenRead(sourceFile))
{
// Upload from the local file to the file share in azure.
await shareFileCopy.CreateAsync(stream.Length);
await shareFileCopy.UploadAsync(stream);
}
Console.WriteLine(" Successfully uploaded file to share.");
// Copy the file to blob storage.
Console.WriteLine("Copying file to blob storage. Container name = {0}", shareName);
// First get a blob service client.
var blobServiceClient = new BlobServiceClient(storageConnectionString);
// Get a blob container client and create it if it doesn't already exist.
blobContainer = blobServiceClient.GetBlobContainerClient(shareName);
await blobContainer.CreateIfNotExistsAsync();
// Get a blob client to the target blob.
targetBlob = blobContainer.GetBlobClient(testFile);
string copyId = string.Empty;
// Get a share file client to be copied.
shareFileClient = fileDirectory.GetFileClient(testFile);
// Create a SAS for the file that's valid for 24 hours.
// Note that when you are copying a file to a blob, or a blob to a file, you must use a SAS
// to authenticate access to the source object, even if you are copying within the same
// storage account.
var sas = new AccountSasBuilder
{
// Allow access to Files
Services = AccountSasServices.Files,
// Allow access to the service level APIs
ResourceTypes = AccountSasResourceTypes.All,
// Access expires in 1 day!
ExpiresOn = DateTime.UtcNow.AddDays(1)
};
sas.SetPermissions(AccountSasPermissions.Read);
var credential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey);
// Build a SAS URI
var sasUri = new UriBuilder(shareFileClient.Uri)
{
Query = sas.ToSasQueryParameters(credential).ToString()
};
// Start the copy of the file to the blob.
CopyFromUriOperation operation = await targetBlob.StartCopyFromUriAsync(sasUri.Uri);
copyId = operation.Id;
Console.WriteLine(" File copy started successfully. copyID = {0}", copyId);
// Now clean up after yourself.
Console.WriteLine("Deleting the files from the file share.");
// Delete the files because cloudFile is a different file in the range sample.
shareFileClient = fileDirectory.GetFileClient(testFile);
await shareFileClient.DeleteIfExistsAsync();
Console.WriteLine("Setting up files to test WriteRange and ListRanges.");
//***** Write 2 ranges to a file, then list the ranges *****//
// This is the code for trying out writing data to a range in a file,
// and then listing those ranges.
// Get a reference to a file and write a range of data to it .
// Then write another range to it.
// Then list the ranges.
// Start at the very beginning of the file.
long startOffset = 0;
// Set the destination file name -- this is the file on the file share that you're writing to.
destFile = "rangeops.txt";
shareFileClient = fileDirectory.GetFileClient(destFile);
// Create a string with 512 a's in it. This will be used to write the range.
int testStreamLen = 512;
string textToStream = string.Empty;
textToStream = textToStream.PadRight(testStreamLen, 'a');
using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream)))
{
// Max size of the output file; have to specify this when you create the file
// I picked this number arbitrarily.
long maxFileSize = 65536;
Console.WriteLine("Write first range.");
// Set the stream back to the beginning, in case it's been read at all.
ms.Position = 0;
// If the file doesn't exist, create it.
// The maximum file size is passed in. It has to be big enough to hold
// all the data you're going to write, so don't set it to 256k and try to write two 256-k blocks to it.
if (!shareFileClient.Exists())
{
Console.WriteLine("File doesn't exist, create empty file to write ranges to.");
// Create a file with a maximum file size of 64k.
await shareFileClient.CreateAsync(maxFileSize);
Console.WriteLine(" Empty file created successfully.");
}
// Write the stream to the file starting at startOffset for the length of the stream.
Console.WriteLine("Writing range to file.");
var range = new HttpRange(startOffset, textToStream.Length);
await shareFileClient.UploadRangeAsync(range, ms);
// Download the file to your temp directory so you can inspect it locally.
downloadFile = Path.Combine(downloadFolder, "__testrange.txt");
Console.WriteLine("Downloading file to examine.");
download = await shareFileClient.DownloadAsync();
using (FileStream stream = File.OpenWrite(downloadFile))
{
await download.Content.CopyToAsync(stream);
}
Console.WriteLine(" Successfully downloaded file with ranges in it to examine.");
}
// Now add the second range, but don't make it adjacent to the first one, or it will show only
// one range, with the two combined. Put it like 1000 spaces away. When you get the range back, it will
// start at the position at the 512-multiple border prior or equal to the beginning of the data written,
// and it will end at the 512-multliple border after the actual end of the data.
//For example, if you write to 2000-3000, the range will be the 512-multiple prior to 2000, which is
// position 1536, or offset 1535 (because it's 0-based).
// And the right offset of the range will be the 512-multiple after 3000, which is position 3072,
// or offset 3071 (because it's 0-based).
Console.WriteLine("Getting ready to write second range to file.");
startOffset += testStreamLen + 1000; //randomly selected number
// Create a string with 512 b's in it. This will be used to write the range.
textToStream = string.Empty;
textToStream = textToStream.PadRight(testStreamLen, 'b');
using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream)))
{
ms.Position = 0;
// Write the stream to the file starting at startOffset for the length of the stream.
Console.WriteLine("Write second range to file.");
var range = new HttpRange(startOffset, textToStream.Length);
await shareFileClient.UploadRangeAsync(range, ms);
Console.WriteLine(" Successful writing second range to file.");
// Download the file to your temp directory so you can examine it.
downloadFile = Path.Combine(downloadFolder, "__testrange2.txt");
Console.WriteLine("Downloading file with two ranges in it to examine.");
download = await shareFileClient.DownloadAsync();
using (FileStream stream = File.OpenWrite(downloadFile))
{
await download.Content.CopyToAsync(stream);
}
Console.WriteLine(" Successfully downloaded file to examine.");
}
// Query and view the list of ranges.
Console.WriteLine("Call to get the list of ranges.");
var listOfRanges = await shareFileClient.GetRangeListAsync(new HttpRange());
Console.WriteLine(" Successfully retrieved list of ranges.");
foreach (HttpRange range in listOfRanges.Value.Ranges)
{
Console.WriteLine(" --> filerange startOffset = {0}, endOffset = {1}", range.Offset, range.Offset + range.Length);
}
//***** Clean up *****//
}
catch (Exception ex)
{
Console.WriteLine(" Exception thrown. Message = {0}{1} Strack Trace = {2}", ex.Message, Environment.NewLine, ex.StackTrace);
}
finally
{
//Clean up after you're done.
Console.WriteLine("Removing all files, folders, shares, blobs, and containers created in this demo.");
// ****NOTE: You can just delete the file share, and everything will be removed.
// This samples deletes everything off of the file share first for the purpose of
// showing you how to delete specific files and directories.
// Delete the file with the ranges in it.
destFile = "rangeops.txt";
shareFileClient = fileDirectory.GetFileClient(destFile);
await shareFileClient.DeleteIfExistsAsync();
Console.WriteLine("Deleting the directory on the file share.");
// Delete the directory.
bool success = await fileDirectory.DeleteIfExistsAsync();
if (success)
{
Console.WriteLine(" Directory on the file share deleted successfully.");
}
else
{
Console.WriteLine(" Directory on the file share NOT deleted successfully; may not exist.");
}
Console.WriteLine("Deleting the file share.");
// Delete the share.
await shareClient.DeleteAsync();
Console.WriteLine(" Deleted the file share successfully.");
Console.WriteLine("Deleting the temporary download directory and the file in it.");
// Delete the download folder and its contents.
Directory.Delete(downloadFolder, true);
Console.WriteLine(" Successfully deleted the temporary download directory.");
Console.WriteLine("Deleting the container and blob used in the Copy/Abort test.");
await targetBlob.DeleteIfExistsAsync();
await blobContainer.DeleteIfExistsAsync();
Console.WriteLine(" Successfully deleted the blob and its container.");
}
}
/// <summary>
/// Calls the 1:1 download method for the given resource client.
/// </summary>
/// <param name="client">Client to call the download on.</param>
/// <param name="destination">Where to send downloaded data.</param>
/// <param name="hashingOptions">Transactional hashing options to use on download.</param>
/// <param name="range">Range parameter for download, necessary for transactional hash request to be accepted by service.</param>
protected abstract Task <Response> DownloadPartitionAsync(
TResourceClient client,
Stream destination,
DownloadTransactionalHashingOptions hashingOptions,
HttpRange range = default);
/// <inheritdoc/>
public async Task <CachedHttpRangeContent> GetOrDownloadContentAsync(Uri blobUri, long desiredOffset)
{
_ = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
if (desiredOffset < 0)
{
throw new ArgumentOutOfRangeException(nameof(desiredOffset), $"Must be greater than zero. {desiredOffset}");
}
// Since the cachedContentTree is only managed on a single-URI basis, we need
// to determine if the cache content is for the requested URI. If not, flush
// out the cache as we're starting over for a new URI.
string uriString = blobUri.ToString();
if (uriString != lastUriCached)
{
CleanUpCachedContentTree();
lastUriCached = uriString;
}
else
{
// It's for the same URI as last call, so check cache for ranges that contain this offset.
var cachedHttpRangeContentEntry = cachedContentTree
.Query((int)desiredOffset)
.OrderByDescending(e => e.CachedHttpRange.Offset)
.FirstOrDefault();
if (cachedHttpRangeContentEntry != default(CachedHttpRangeContent))
{
var message = $"Found desiredOffset, {desiredOffset}, in existing range.";
_log.LogInformation(message);
return(cachedHttpRangeContentEntry);
}
}
// No luck, nothing suitable in the cache so we're going to have to download a new range to cover
// the request. Clean out the whole tree if we are about to exceed the MaxMemorySize.
if (totalContentLength + DefaultLength >= MaxCachedBytes)
{
CleanUpCachedContentTree();
}
int downloadedContentLength = 0;
MemoryStream memStream = null;
var requestedHttpRange = new HttpRange(desiredOffset, DefaultLength);
try
{
using var downloadResponse = await DownloadHttpRangeAsync(blobUri, requestedHttpRange).ConfigureAwait(false);
downloadedContentLength = (int)downloadResponse.ContentLength;
totalContentLength += downloadedContentLength;
#pragma warning disable CA2000 // Dispose objects before losing scope
memStream = new MemoryStream(downloadedContentLength);
#pragma warning restore CA2000 // Dispose objects before losing scope
downloadResponse.Content.CopyTo(memStream);
}
catch (Exception e) when(
e is ArgumentOutOfRangeException ||
e is ArgumentNullException ||
e is NotSupportedException ||
e is ObjectDisposedException ||
e is IOException)
{
var message = $"Could not download content for {blobUri}.";
_log.LogError(message, e);
throw new Exception(message, e);
}
var actualHttpRange = new HttpRange(desiredOffset, downloadedContentLength);
var cachedHttpRangeContent = new CachedHttpRangeContent(actualHttpRange, memStream);
cachedContentTree.Add((int)actualHttpRange.Offset, (int)(actualHttpRange.Offset + actualHttpRange.Length - 1), cachedHttpRangeContent);
// Console.WriteLine($"Added Range:\t\t {actualHttpRange.Offset},\t\t {actualHttpRange.Offset + actualHttpRange.Length - 1}");
_log.LogInformation($"HttpRangeDownloadedFinished, {actualHttpRange.Offset}, {actualHttpRange.Length}");
return(cachedHttpRangeContent);
}
public async Task <Response> DownloadToAsync(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}");
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Task <Response <BlobDownloadInfo> > initialResponseTask =
_client.DownloadAsync(
initialRange,
conditions,
rangeGetContentHash: false,
cancellationToken);
Response <BlobDownloadInfo> initialResponse =
await initialResponseTask.ConfigureAwait(false);
// If the initial request returned no content (i.e., a 304),
// we'll pass that back to the user immediately
if (initialResponse.IsUnavailable())
{
return(initialResponse.GetRawResponse());
}
// If the first segment was the entire blob, we'll copy that to
// the output stream and finish now
long initialLength = initialResponse.Value.ContentLength;
long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange);
if (initialLength == totalLength)
{
await CopyToAsync(
initialResponse,
destination,
cancellationToken)
.ConfigureAwait(false);
return(initialResponse.GetRawResponse());
}
// Capture the etag from the first segment and construct
// conditions to ensure the blob doesn't change while we're
// downloading the remaining segments
ETag etag = initialResponse.Value.Details.ETag;
BlobRequestConditions conditionsWithEtag = CreateConditionsWithEtag(conditions, etag);
// Create a queue of tasks that will each download one segment
// of the blob. The queue maintains the order of the segments
// so we can keep appending to the end of the destination
// stream when each segment finishes.
var runningTasks = new Queue <Task <Response <BlobDownloadInfo> > >();
runningTasks.Enqueue(initialResponseTask);
// Fill the queue with tasks to download each of the remaining
// ranges in the blob
foreach (HttpRange httpRange in GetRanges(initialLength, totalLength))
{
// Add the next Task (which will start the download but
// return before it's completed downloading)
runningTasks.Enqueue(_client.DownloadAsync(
httpRange,
conditionsWithEtag,
rangeGetContentHash: false,
cancellationToken));
// If we have fewer tasks than alotted workers, then just
// continue adding tasks until we have _maxWorkerCount
// running in parallel
if (runningTasks.Count < _maxWorkerCount)
{
continue;
}
// Once all the workers are busy, wait for the first
// segment to finish downloading before we create more work
await ConsumeQueuedTask().ConfigureAwait(false);
}
// Wait for all of the remaining segments to download
while (runningTasks.Count > 0)
{
await ConsumeQueuedTask().ConfigureAwait(false);
}
return(initialResponse.GetRawResponse());
// Wait for the first segment in the queue of tasks to complete
// downloading and copy it to the destination stream
async Task ConsumeQueuedTask()
{
// Don't need to worry about 304s here because the ETag
// condition will turn into a 412 and throw a proper
// RequestFailedException
using BlobDownloadInfo result =
await runningTasks.Dequeue().ConfigureAwait(false);
// Even though the BlobDownloadInfo is returned immediately,
// CopyToAsync causes ConsumeQueuedTask to wait until the
// download is complete
await CopyToAsync(
result,
destination,
cancellationToken)
.ConfigureAwait(false);
}
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
private async Task <int> DownloadInternal(bool async, CancellationToken cancellationToken)
{
Response <IDownloadedContent> response;
HttpRange range = new HttpRange(_position, _bufferSize);
// if _downloadInternalFunc is going to produce a range out of bounds response, we're at the end of the blob
if (_predictEncryptedRangeAdjustment(range).Offset >= _length)
{
return(0);
}
// TODO #27253
response = await _downloadInternalFunc(range, /*_hashingOptions,*/ async, cancellationToken).ConfigureAwait(false);
using Stream networkStream = response.Value.Content;
// The number of bytes we just downloaded.
long downloadSize = GetResponseRange(response.GetRawResponse()).Length.Value;
// The number of bytes we copied in the last loop.
int copiedBytes;
// Bytes we have copied so far.
int totalCopiedBytes = 0;
// Bytes remaining to copy. It is save to truncate the long because we asked for a max of int _buffer size bytes.
int remainingBytes = (int)downloadSize;
do
{
if (async)
{
copiedBytes = await networkStream.ReadAsync(
buffer : _buffer,
offset : totalCopiedBytes,
count : remainingBytes,
cancellationToken : cancellationToken).ConfigureAwait(false);
}
else
{
copiedBytes = networkStream.Read(
buffer: _buffer,
offset: totalCopiedBytes,
count: remainingBytes);
}
totalCopiedBytes += copiedBytes;
remainingBytes -= copiedBytes;
}while (copiedBytes != 0);
_bufferPosition = 0;
_bufferLength = totalCopiedBytes;
_length = GetBlobLengthFromResponse(response.GetRawResponse());
// TODO #27253
// if we deferred transactional hash validation on download, validate now
// currently we always do but that may change
//if (_hashingOptions != default && !_hashingOptions.Validate)
//{
// ContentHasher.AssertResponseHashMatch(_buffer, _bufferPosition, _bufferLength, _hashingOptions.Algorithm, response.GetRawResponse());
//}
return(totalCopiedBytes);
}
internal async Task <Response <Stream> > DownloadStreamingInternal(Uri sourceEndpoint, HttpRange range, bool async, CancellationToken cancellationToken)
{
Argument.AssertNotNull(sourceEndpoint, nameof(sourceEndpoint));
Response <Stream> response;
if (async)
{
response = await StartDownloadAsync(sourceEndpoint, range, cancellationToken : cancellationToken).ConfigureAwait(false);
}
else
{
response = StartDownload(sourceEndpoint, range, cancellationToken: cancellationToken);
}
Stream stream = RetriableStream.Create(
response.Value,
startOffset => StartDownload(sourceEndpoint, range, startOffset, cancellationToken).Value,
async startOffset => (await StartDownloadAsync(sourceEndpoint, range, startOffset, cancellationToken).ConfigureAwait(false)).Value,
_client._pipeline.ResponseClassifier,
Constants.ContentDownloader.RetriableStreamRetries
);
return(Response.FromValue(stream, response.GetRawResponse()));
}
/// <inheritdoc/>
public async Task <BlobDownloadInfo> DownloadHttpRangeAsync(Uri blobUri, StorageClientProviderContext context, HttpRange httpRange = default)
{
_ = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
_ = context ?? throw new ArgumentNullException(nameof(context));
// Note: when the httpRange struct is omitted it defaults to 'default' which downloads the entire blob.
IStorageBlobClientSleeve blobBaseClient = _blobBaseClientProvider.GetBlobClientSleeveForUri(blobUri, context);
BlobDownloadInfo blobDownloadInfo;
try
{
blobDownloadInfo = (await blobBaseClient.Client.DownloadAsync(httpRange).ConfigureAwait(false)).Value;
}
catch (Exception e)
{
_log.LogExceptionObject(LogEventIds.FailedToDownloadHttpRangeInStorageService, e, blobUri);
throw new GridwichStorageServiceException(blobUri, "Could not download the HTTP range for a blob.",
LogEventIds.FailedToDownloadHttpRangeInStorageService, context.ClientRequestIdAsJObject, e);
}
return(blobDownloadInfo);
}
private async Task <Response <Stream> > StartDownloadAsync(Uri sourceEndpoint, HttpRange range = default, long startOffset = 0,
CancellationToken cancellationToken = default)
{
HttpRange?pageRange = null;
if (range != default || startOffset != 0)
{
pageRange = new HttpRange(
range.Offset + startOffset,
range.Length.HasValue ?
range.Length.Value - startOffset :
null);
}
HttpMessage message = GetHttpMessage(sourceEndpoint, pageRange);
await _client._pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false);
switch (message.Response.Status)
{
case 200:
case 206:
{
Stream value = message.ExtractResponseContent();
return(Response.FromValue(value, message.Response));
}
default:
throw await _client._clientDiagnostics.CreateRequestFailedExceptionAsync(message.Response).ConfigureAwait(false);
}
}
public Response DownloadTo(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}");
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Response <BlobDownloadInfo> initialResponse = _client.Download(
initialRange,
conditions,
rangeGetContentHash: false,
cancellationToken);
// If the initial request returned no content (i.e., a 304),
// we'll pass that back to the user immediately
if (initialResponse.IsUnavailable())
{
return(initialResponse.GetRawResponse());
}
// Copy the first segment to the destination stream
CopyTo(initialResponse, destination, cancellationToken);
// If the first segment was the entire blob, we're finished now
long initialLength = initialResponse.Value.ContentLength;
long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange);
if (initialLength == totalLength)
{
return(initialResponse.GetRawResponse());
}
// Capture the etag from the first segment and construct
// conditions to ensure the blob doesn't change while we're
// downloading the remaining segments
ETag etag = initialResponse.Value.Details.ETag;
BlobRequestConditions conditionsWithEtag = CreateConditionsWithEtag(conditions, etag);
// Download each of the remaining ranges in the blob
foreach (HttpRange httpRange in GetRanges(initialLength, totalLength))
{
// Don't need to worry about 304s here because the ETag
// condition will turn into a 412 and throw a proper
// RequestFailedException
Response <BlobDownloadInfo> result = _client.Download(
httpRange,
conditionsWithEtag,
rangeGetContentHash: false,
cancellationToken);
CopyTo(result.Value, destination, cancellationToken);
}
return(initialResponse.GetRawResponse());
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
