public async Task <Response> DownloadToAsync( Stream destination, BlobRequestConditions conditions, CancellationToken cancellationToken) { // Wrap the download range calls in a Download span for distributed // tracing DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}"); try { scope.Start(); // Just start downloading using an initial range. If it's a // small blob, we'll get the whole thing in one shot. If it's // a large blob, we'll get its full size in Content-Range and // can keep downloading it in segments. var initialRange = new HttpRange(0, _initialRangeSize); Task <Response <BlobDownloadInfo> > initialResponseTask = _client.DownloadAsync( initialRange, conditions, rangeGetContentHash: false, cancellationToken); Response <BlobDownloadInfo> initialResponse = null; try { initialResponse = await initialResponseTask.ConfigureAwait(false); } catch (RequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.InvalidRange) { initialResponse = await _client.DownloadAsync( range : default,
public static async Task <string> DownloadTextAsync(this BlobBaseClient blobClient, CancellationToken cancellationToken = default) { using BlobDownloadInfo blobDownloadInfo = await blobClient.DownloadAsync(cancellationToken).ConfigureAwait(false); using Stream stream = blobDownloadInfo.Content; using StreamReader streamReader = new StreamReader(stream); return(await streamReader.ReadToEndAsync().ConfigureAwait(false)); }
public async Task SetImmutibilityPolicyAsync() { // Arrange await using DisposingImmutableStorageWithVersioningContainer vlwContainer = await GetTestVersionLevelWormContainer(TestConfigOAuth); BlobBaseClient blob = await GetNewBlobClient(vlwContainer.Container); BlobImmutabilityPolicy immutabilityPolicy = new BlobImmutabilityPolicy { ExpiresOn = Recording.UtcNow.AddMinutes(5), PolicyMode = BlobImmutabilityPolicyMode.Unlocked }; // The service rounds Immutability Policy Expiry to the nearest second. DateTimeOffset expectedImmutabilityPolicyExpiry = RoundToNearestSecond(immutabilityPolicy.ExpiresOn.Value); // Test SetImmutabilityPolicyAsync API and validate response. // Act Response <BlobImmutabilityPolicy> response = await blob.SetImmutabilityPolicyAsync(immutabilityPolicy); // Assert Assert.AreEqual(expectedImmutabilityPolicyExpiry, response.Value.ExpiresOn); Assert.AreEqual(immutabilityPolicy.PolicyMode, response.Value.PolicyMode); // Validate that we are correctly deserializing Get Properties response. // Act Response <BlobProperties> propertiesResponse = await blob.GetPropertiesAsync(); // Assert Assert.AreEqual(expectedImmutabilityPolicyExpiry, propertiesResponse.Value.ImmutabilityPolicy.ExpiresOn); Assert.AreEqual(immutabilityPolicy.PolicyMode, propertiesResponse.Value.ImmutabilityPolicy.PolicyMode); // Validate we are correctly deserializing Blob Items. // Act List <BlobItem> blobItems = new List <BlobItem>(); await foreach (BlobItem blobItem in vlwContainer.Container.GetBlobsAsync(traits: BlobTraits.ImmutabilityPolicy)) { blobItems.Add(blobItem); } // Assert Assert.AreEqual(1, blobItems.Count); Assert.AreEqual(expectedImmutabilityPolicyExpiry, blobItems[0].Properties.ImmutabilityPolicy.ExpiresOn); Assert.AreEqual(immutabilityPolicy.PolicyMode, blobItems[0].Properties.ImmutabilityPolicy.PolicyMode); // Validate we are correctly deserialzing Get Blob response. // Act Response <BlobDownloadInfo> downloadResponse = await blob.DownloadAsync(); // Assert Assert.AreEqual(expectedImmutabilityPolicyExpiry, downloadResponse.Value.Details.ImmutabilityPolicy.ExpiresOn); Assert.AreEqual(immutabilityPolicy.PolicyMode, downloadResponse.Value.Details.ImmutabilityPolicy.PolicyMode); }
/// <summary> /// Given a log file (as a blob), parses it and return a collection of log entries matching version 1.0 /// of the Storage Analytics Log Format. /// </summary> /// <param name="blob">Object representing a cloud blob with Storage Analytics log content.</param> /// <param name="cancellationToken">A token to monitor for cancellation request.</param> /// <returns>Collection of successfully parsed log entries.</returns> /// <exception cref="FormatException">If unable to parse a line in given log.</exception> /// <seealso cref="StorageAnalyticsLogEntry"/> /// <remarks> /// The method scans log file lines one at a time. /// First it attempts to detect a line format version and throws an exception if failed to do so. /// It skips all lines with version different than supported one, i.e. 1.0. /// Then it calls TryParseLogEntry to create a log entry out of every line of supported version and throws /// an exception if the parse method returns null. /// </remarks> public async Task <IEnumerable <StorageAnalyticsLogEntry> > ParseLogAsync(BlobBaseClient blob, CancellationToken cancellationToken) { List <StorageAnalyticsLogEntry> entries = new List <StorageAnalyticsLogEntry>(); using (Stream content = (await blob.DownloadAsync(cancellationToken).ConfigureAwait(false)).Value.Content) { using (TextReader tr = new StreamReader(content)) { for (int lineNumber = 1; ; lineNumber++) { cancellationToken.ThrowIfCancellationRequested(); string line = await tr.ReadLineAsync().ConfigureAwait(false); if (line == null) { break; } Version version = TryParseVersion(line); if (version == null) { string message = String.Format(CultureInfo.CurrentCulture, "Unable to detect a version of log entry on line {1} of Storage Analytics log file '{0}'.", blob.Name, lineNumber); _logger.LogWarning(message); } if (version == supportedVersion) { StorageAnalyticsLogEntry entry = TryParseLogEntry(line); if (entry == null) { string message = String.Format(CultureInfo.CurrentCulture, "Unable to parse the log entry on line {1} of Storage Analytics log file '{0}'.", blob.Name, lineNumber); _logger.LogWarning(message); } entries.Add(entry); } } } } return(entries); }
public async Task SetLegalHoldAsync() { // Arrange await using DisposingImmutableStorageWithVersioningContainer vlwContainer = await GetTestVersionLevelWormContainer(TestConfigOAuth); BlobBaseClient blob = await GetNewBlobClient(vlwContainer.Container); // Act Response <BlobLegalHoldResult> response = await blob.SetLegalHoldAsync(true); // Assert Assert.IsTrue(response.Value.HasLegalHold); // Validate that we are correctly deserializing Get Properties response. // Act Response <BlobProperties> propertiesResponse = await blob.GetPropertiesAsync(); // Assert Assert.IsTrue(propertiesResponse.Value.HasLegalHold); // Validate we are correctly deserializing Blob Items. // Act List <BlobItem> blobItems = new List <BlobItem>(); await foreach (BlobItem blobItem in vlwContainer.Container.GetBlobsAsync(traits: BlobTraits.LegalHold)) { blobItems.Add(blobItem); } // Assert Assert.AreEqual(1, blobItems.Count); Assert.IsTrue(blobItems[0].Properties.HasLegalHold); // Validate we are correctly deserialzing Get Blob response. // Act Response <BlobDownloadInfo> downloadResponse = await blob.DownloadAsync(); // Assert Assert.IsTrue(downloadResponse.Value.Details.HasLegalHold); // Act response = await blob.SetLegalHoldAsync(false); // Assert Assert.IsFalse(response.Value.HasLegalHold); }
public async Task SetLegalHoldAsync() { // Arrange BlobBaseClient blob = await GetNewBlobClient(_containerClient); // Act Response <BlobLegalHoldResult> response = await blob.SetLegalHoldAsync(true); // Assert Assert.IsTrue(response.Value.HasLegalHold); // Validate that we are correctly deserializing Get Properties response. // Act Response <BlobProperties> propertiesResponse = await blob.GetPropertiesAsync(); // Assert Assert.IsTrue(propertiesResponse.Value.HasLegalHold); // Validate we are correctly deserializing Blob Items. // Act List <BlobItem> blobItems = new List <BlobItem>(); await foreach (BlobItem blobItem in _containerClient.GetBlobsAsync(traits: BlobTraits.LegalHold, prefix: blob.Name)) { blobItems.Add(blobItem); } // Assert Assert.AreEqual(1, blobItems.Count); Assert.IsTrue(blobItems[0].Properties.HasLegalHold); // Validate we are correctly deserialzing Get Blob response. // Act Response <BlobDownloadInfo> downloadResponse = await blob.DownloadAsync(); // Assert Assert.IsTrue(downloadResponse.Value.Details.HasLegalHold); // Act response = await blob.SetLegalHoldAsync(false); // Assert Assert.IsFalse(response.Value.HasLegalHold); }
/// <inheritdoc/> public async Task <BlobDownloadInfo> DownloadHttpRangeAsync(Uri blobUri, HttpRange httpRange = default) { _ = blobUri ?? throw new ArgumentNullException(nameof(blobUri)); var blobBaseClient = new BlobBaseClient(blobUri, _tokenCredential); // Note: when the httpRange struct is omitted it defaults to 'default' which downloads the entire blob. BlobDownloadInfo blobDownloadInfo; try { blobDownloadInfo = (await blobBaseClient.DownloadAsync(httpRange).ConfigureAwait(false)).Value; } catch (Exception e) { var message = $"Could not download the HTTP range for {blobUri}."; _log.LogError(message, e); throw new Exception(message, e); } return(blobDownloadInfo); }
public async Task <Response> DownloadToAsync( Stream destination, BlobRequestConditions conditions, CancellationToken cancellationToken) { // Wrap the download range calls in a Download span for distributed // tracing DiagnosticScope scope = _client.ClientDiagnostics.CreateScope($"{nameof(BlobBaseClient)}.{nameof(BlobBaseClient.DownloadTo)}"); try { scope.Start(); // Just start downloading using an initial range. If it's a // small blob, we'll get the whole thing in one shot. If it's // a large blob, we'll get its full size in Content-Range and // can keep downloading it in segments. var initialRange = new HttpRange(0, _initialRangeSize); Task <Response <BlobDownloadInfo> > initialResponseTask = _client.DownloadAsync( initialRange, conditions, rangeGetContentHash: false, cancellationToken); Response <BlobDownloadInfo> initialResponse = await initialResponseTask.ConfigureAwait(false); // If the initial request returned no content (i.e., a 304), // we'll pass that back to the user immediately if (initialResponse.IsUnavailable()) { return(initialResponse.GetRawResponse()); } // If the first segment was the entire blob, we'll copy that to // the output stream and finish now long initialLength = initialResponse.Value.ContentLength; long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange); if (initialLength == totalLength) { await CopyToAsync( initialResponse, destination, cancellationToken) .ConfigureAwait(false); return(initialResponse.GetRawResponse()); } // Capture the etag from the first segment and construct // conditions to ensure the blob doesn't change while we're // downloading the remaining segments ETag etag = initialResponse.Value.Details.ETag; BlobRequestConditions conditionsWithEtag = CreateConditionsWithEtag(conditions, etag); // Create a queue of tasks that will each download one segment // of the blob. The queue maintains the order of the segments // so we can keep appending to the end of the destination // stream when each segment finishes. var runningTasks = new Queue <Task <Response <BlobDownloadInfo> > >(); runningTasks.Enqueue(initialResponseTask); // Fill the queue with tasks to download each of the remaining // ranges in the blob foreach (HttpRange httpRange in GetRanges(initialLength, totalLength)) { // Add the next Task (which will start the download but // return before it's completed downloading) runningTasks.Enqueue(_client.DownloadAsync( httpRange, conditionsWithEtag, rangeGetContentHash: false, cancellationToken)); // If we have fewer tasks than alotted workers, then just // continue adding tasks until we have _maxWorkerCount // running in parallel if (runningTasks.Count < _maxWorkerCount) { continue; } // Once all the workers are busy, wait for the first // segment to finish downloading before we create more work await ConsumeQueuedTask().ConfigureAwait(false); } // Wait for all of the remaining segments to download while (runningTasks.Count > 0) { await ConsumeQueuedTask().ConfigureAwait(false); } return(initialResponse.GetRawResponse()); // Wait for the first segment in the queue of tasks to complete // downloading and copy it to the destination stream async Task ConsumeQueuedTask() { // Don't need to worry about 304s here because the ETag // condition will turn into a 412 and throw a proper // RequestFailedException using BlobDownloadInfo result = await runningTasks.Dequeue().ConfigureAwait(false); // Even though the BlobDownloadInfo is returned immediately, // CopyToAsync causes ConsumeQueuedTask to wait until the // download is complete await CopyToAsync( result, destination, cancellationToken) .ConfigureAwait(false); } } catch (Exception ex) { scope.Failed(ex); throw; } finally { scope.Dispose(); } }