public async Task <Response> DownloadToAsync(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}");
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Task <Response <BlobDownloadInfo> > initialResponseTask =
_client.DownloadAsync(
initialRange,
conditions,
rangeGetContentHash: false,
cancellationToken);
Response <BlobDownloadInfo> initialResponse = null;
try
{
initialResponse = await initialResponseTask.ConfigureAwait(false);
}
catch (RequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.InvalidRange)
{
initialResponse = await _client.DownloadAsync(
range : default,
public static async Task <string> DownloadTextAsync(this BlobBaseClient blobClient, CancellationToken cancellationToken = default)
{
using BlobDownloadInfo blobDownloadInfo = await blobClient.DownloadAsync(cancellationToken).ConfigureAwait(false);
using Stream stream = blobDownloadInfo.Content;
using StreamReader streamReader = new StreamReader(stream);
return(await streamReader.ReadToEndAsync().ConfigureAwait(false));
}
public async Task SetImmutibilityPolicyAsync()
{
// Arrange
await using DisposingImmutableStorageWithVersioningContainer vlwContainer = await GetTestVersionLevelWormContainer(TestConfigOAuth);
BlobBaseClient blob = await GetNewBlobClient(vlwContainer.Container);
BlobImmutabilityPolicy immutabilityPolicy = new BlobImmutabilityPolicy
{
ExpiresOn = Recording.UtcNow.AddMinutes(5),
PolicyMode = BlobImmutabilityPolicyMode.Unlocked
};
// The service rounds Immutability Policy Expiry to the nearest second.
DateTimeOffset expectedImmutabilityPolicyExpiry = RoundToNearestSecond(immutabilityPolicy.ExpiresOn.Value);
// Test SetImmutabilityPolicyAsync API and validate response.
// Act
Response <BlobImmutabilityPolicy> response = await blob.SetImmutabilityPolicyAsync(immutabilityPolicy);
// Assert
Assert.AreEqual(expectedImmutabilityPolicyExpiry, response.Value.ExpiresOn);
Assert.AreEqual(immutabilityPolicy.PolicyMode, response.Value.PolicyMode);
// Validate that we are correctly deserializing Get Properties response.
// Act
Response <BlobProperties> propertiesResponse = await blob.GetPropertiesAsync();
// Assert
Assert.AreEqual(expectedImmutabilityPolicyExpiry, propertiesResponse.Value.ImmutabilityPolicy.ExpiresOn);
Assert.AreEqual(immutabilityPolicy.PolicyMode, propertiesResponse.Value.ImmutabilityPolicy.PolicyMode);
// Validate we are correctly deserializing Blob Items.
// Act
List <BlobItem> blobItems = new List <BlobItem>();
await foreach (BlobItem blobItem in vlwContainer.Container.GetBlobsAsync(traits: BlobTraits.ImmutabilityPolicy))
{
blobItems.Add(blobItem);
}
// Assert
Assert.AreEqual(1, blobItems.Count);
Assert.AreEqual(expectedImmutabilityPolicyExpiry, blobItems[0].Properties.ImmutabilityPolicy.ExpiresOn);
Assert.AreEqual(immutabilityPolicy.PolicyMode, blobItems[0].Properties.ImmutabilityPolicy.PolicyMode);
// Validate we are correctly deserialzing Get Blob response.
// Act
Response <BlobDownloadInfo> downloadResponse = await blob.DownloadAsync();
// Assert
Assert.AreEqual(expectedImmutabilityPolicyExpiry, downloadResponse.Value.Details.ImmutabilityPolicy.ExpiresOn);
Assert.AreEqual(immutabilityPolicy.PolicyMode, downloadResponse.Value.Details.ImmutabilityPolicy.PolicyMode);
}
/// <summary>
/// Given a log file (as a blob), parses it and return a collection of log entries matching version 1.0
/// of the Storage Analytics Log Format.
/// </summary>
/// <param name="blob">Object representing a cloud blob with Storage Analytics log content.</param>
/// <param name="cancellationToken">A token to monitor for cancellation request.</param>
/// <returns>Collection of successfully parsed log entries.</returns>
/// <exception cref="FormatException">If unable to parse a line in given log.</exception>
/// <seealso cref="StorageAnalyticsLogEntry"/>
/// <remarks>
/// The method scans log file lines one at a time.
/// First it attempts to detect a line format version and throws an exception if failed to do so.
/// It skips all lines with version different than supported one, i.e. 1.0.
/// Then it calls TryParseLogEntry to create a log entry out of every line of supported version and throws
/// an exception if the parse method returns null.
/// </remarks>
public async Task <IEnumerable <StorageAnalyticsLogEntry> > ParseLogAsync(BlobBaseClient blob,
CancellationToken cancellationToken)
{
List <StorageAnalyticsLogEntry> entries = new List <StorageAnalyticsLogEntry>();
using (Stream content = (await blob.DownloadAsync(cancellationToken).ConfigureAwait(false)).Value.Content)
{
using (TextReader tr = new StreamReader(content))
{
for (int lineNumber = 1; ; lineNumber++)
{
cancellationToken.ThrowIfCancellationRequested();
string line = await tr.ReadLineAsync().ConfigureAwait(false);
if (line == null)
{
break;
}
Version version = TryParseVersion(line);
if (version == null)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Unable to detect a version of log entry on line {1} of Storage Analytics log file '{0}'.",
blob.Name, lineNumber);
_logger.LogWarning(message);
}
if (version == supportedVersion)
{
StorageAnalyticsLogEntry entry = TryParseLogEntry(line);
if (entry == null)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Unable to parse the log entry on line {1} of Storage Analytics log file '{0}'.",
blob.Name, lineNumber);
_logger.LogWarning(message);
}
entries.Add(entry);
}
}
}
}
return(entries);
}
public async Task SetLegalHoldAsync()
{
// Arrange
await using DisposingImmutableStorageWithVersioningContainer vlwContainer = await GetTestVersionLevelWormContainer(TestConfigOAuth);
BlobBaseClient blob = await GetNewBlobClient(vlwContainer.Container);
// Act
Response <BlobLegalHoldResult> response = await blob.SetLegalHoldAsync(true);
// Assert
Assert.IsTrue(response.Value.HasLegalHold);
// Validate that we are correctly deserializing Get Properties response.
// Act
Response <BlobProperties> propertiesResponse = await blob.GetPropertiesAsync();
// Assert
Assert.IsTrue(propertiesResponse.Value.HasLegalHold);
// Validate we are correctly deserializing Blob Items.
// Act
List <BlobItem> blobItems = new List <BlobItem>();
await foreach (BlobItem blobItem in vlwContainer.Container.GetBlobsAsync(traits: BlobTraits.LegalHold))
{
blobItems.Add(blobItem);
}
// Assert
Assert.AreEqual(1, blobItems.Count);
Assert.IsTrue(blobItems[0].Properties.HasLegalHold);
// Validate we are correctly deserialzing Get Blob response.
// Act
Response <BlobDownloadInfo> downloadResponse = await blob.DownloadAsync();
// Assert
Assert.IsTrue(downloadResponse.Value.Details.HasLegalHold);
// Act
response = await blob.SetLegalHoldAsync(false);
// Assert
Assert.IsFalse(response.Value.HasLegalHold);
}
public async Task SetLegalHoldAsync()
{
// Arrange
BlobBaseClient blob = await GetNewBlobClient(_containerClient);
// Act
Response <BlobLegalHoldResult> response = await blob.SetLegalHoldAsync(true);
// Assert
Assert.IsTrue(response.Value.HasLegalHold);
// Validate that we are correctly deserializing Get Properties response.
// Act
Response <BlobProperties> propertiesResponse = await blob.GetPropertiesAsync();
// Assert
Assert.IsTrue(propertiesResponse.Value.HasLegalHold);
// Validate we are correctly deserializing Blob Items.
// Act
List <BlobItem> blobItems = new List <BlobItem>();
await foreach (BlobItem blobItem in _containerClient.GetBlobsAsync(traits: BlobTraits.LegalHold, prefix: blob.Name))
{
blobItems.Add(blobItem);
}
// Assert
Assert.AreEqual(1, blobItems.Count);
Assert.IsTrue(blobItems[0].Properties.HasLegalHold);
// Validate we are correctly deserialzing Get Blob response.
// Act
Response <BlobDownloadInfo> downloadResponse = await blob.DownloadAsync();
// Assert
Assert.IsTrue(downloadResponse.Value.Details.HasLegalHold);
// Act
response = await blob.SetLegalHoldAsync(false);
// Assert
Assert.IsFalse(response.Value.HasLegalHold);
}
/// <inheritdoc/>
public async Task <BlobDownloadInfo> DownloadHttpRangeAsync(Uri blobUri, HttpRange httpRange = default)
{
_ = blobUri ?? throw new ArgumentNullException(nameof(blobUri));
var blobBaseClient = new BlobBaseClient(blobUri, _tokenCredential);
// Note: when the httpRange struct is omitted it defaults to 'default' which downloads the entire blob.
BlobDownloadInfo blobDownloadInfo;
try
{
blobDownloadInfo = (await blobBaseClient.DownloadAsync(httpRange).ConfigureAwait(false)).Value;
}
catch (Exception e)
{
var message = $"Could not download the HTTP range for {blobUri}.";
_log.LogError(message, e);
throw new Exception(message, e);
}
return(blobDownloadInfo);
}
public async Task <Response> DownloadToAsync(
Stream destination,
BlobRequestConditions conditions,
CancellationToken cancellationToken)
{
// Wrap the download range calls in a Download span for distributed
// tracing
DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}");
try
{
scope.Start();
// Just start downloading using an initial range. If it's a
// small blob, we'll get the whole thing in one shot. If it's
// a large blob, we'll get its full size in Content-Range and
// can keep downloading it in segments.
var initialRange = new HttpRange(0, _initialRangeSize);
Task <Response <BlobDownloadInfo> > initialResponseTask =
_client.DownloadAsync(
initialRange,
conditions,
rangeGetContentHash: false,
cancellationToken);
Response <BlobDownloadInfo> initialResponse =
await initialResponseTask.ConfigureAwait(false);
// If the initial request returned no content (i.e., a 304),
// we'll pass that back to the user immediately
if (initialResponse.IsUnavailable())
{
return(initialResponse.GetRawResponse());
}
// If the first segment was the entire blob, we'll copy that to
// the output stream and finish now
long initialLength = initialResponse.Value.ContentLength;
long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange);
if (initialLength == totalLength)
{
await CopyToAsync(
initialResponse,
destination,
cancellationToken)
.ConfigureAwait(false);
return(initialResponse.GetRawResponse());
}
// Capture the etag from the first segment and construct
// conditions to ensure the blob doesn't change while we're
// downloading the remaining segments
ETag etag = initialResponse.Value.Details.ETag;
BlobRequestConditions conditionsWithEtag = CreateConditionsWithEtag(conditions, etag);
// Create a queue of tasks that will each download one segment
// of the blob. The queue maintains the order of the segments
// so we can keep appending to the end of the destination
// stream when each segment finishes.
var runningTasks = new Queue <Task <Response <BlobDownloadInfo> > >();
runningTasks.Enqueue(initialResponseTask);
// Fill the queue with tasks to download each of the remaining
// ranges in the blob
foreach (HttpRange httpRange in GetRanges(initialLength, totalLength))
{
// Add the next Task (which will start the download but
// return before it's completed downloading)
runningTasks.Enqueue(_client.DownloadAsync(
httpRange,
conditionsWithEtag,
rangeGetContentHash: false,
cancellationToken));
// If we have fewer tasks than alotted workers, then just
// continue adding tasks until we have _maxWorkerCount
// running in parallel
if (runningTasks.Count < _maxWorkerCount)
{
continue;
}
// Once all the workers are busy, wait for the first
// segment to finish downloading before we create more work
await ConsumeQueuedTask().ConfigureAwait(false);
}
// Wait for all of the remaining segments to download
while (runningTasks.Count > 0)
{
await ConsumeQueuedTask().ConfigureAwait(false);
}
return(initialResponse.GetRawResponse());
// Wait for the first segment in the queue of tasks to complete
// downloading and copy it to the destination stream
async Task ConsumeQueuedTask()
{
// Don't need to worry about 304s here because the ETag
// condition will turn into a 412 and throw a proper
// RequestFailedException
using BlobDownloadInfo result =
await runningTasks.Dequeue().ConfigureAwait(false);
// Even though the BlobDownloadInfo is returned immediately,
// CopyToAsync causes ConsumeQueuedTask to wait until the
// download is complete
await CopyToAsync(
result,
destination,
cancellationToken)
.ConfigureAwait(false);
}
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
finally
{
scope.Dispose();
}
}
