public void BatchErrors() { string connectionString = ConnectionString; string containerName = Randomize("sample-container"); #region Snippet:SampleSnippetsBatch_Troubleshooting // Get a connection string to our Azure Storage account. //@@ string connectionString = "<connection_string>"; //@@ string containerName = "sample-container"; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(containerName); container.Create(); // Create a blob named "valid" BlobClient valid = container.GetBlobClient("valid"); valid.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Valid!"))); // Get a reference to a blob named "invalid", but never create it BlobClient invalid = container.GetBlobClient("invalid"); // Delete both blobs at the same time BlobBatchClient batch = service.GetBlobBatchClient(); try { batch.DeleteBlobs(new Uri[] { valid.Uri, invalid.Uri }); } catch (AggregateException) { // An aggregate exception is thrown for all the individual failures // Check ex.InnerExceptions for RequestFailedException instances } #endregion }
public void BatchDelete() { string connectionString = ConnectionString; string containerName = Randomize("sample-container"); #region Snippet:SampleSnippetsBatch_DeleteBatch // Get a connection string to our Azure Storage account. //@@ string connectionString = "<connection_string>"; //@@ string containerName = "sample-container"; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(containerName); container.Create(); // Create three blobs named "foo", "bar", and "baz" BlobClient foo = container.GetBlobClient("foo"); BlobClient bar = container.GetBlobClient("bar"); BlobClient baz = container.GetBlobClient("baz"); foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!"))); bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!"))); baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!"))); // Delete all three blobs at once BlobBatchClient batch = service.GetBlobBatchClient(); batch.DeleteBlobs(new Uri[] { foo.Uri, bar.Uri, baz.Uri }); #endregion Assert.AreEqual(0, container.GetBlobs().ToList().Count); // Clean up after we're finished container.Delete(); }
public async Task BatchSetAccessTierAsync() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container")); await container.CreateAsync(); try { // Create three blobs named "foo", "bar", and "baz" BlobClient foo = container.GetBlobClient("foo"); BlobClient bar = container.GetBlobClient("bar"); BlobClient baz = container.GetBlobClient("baz"); await foo.UploadAsync(BinaryData.FromString("Foo!")); await bar.UploadAsync(BinaryData.FromString("Bar!")); await baz.UploadAsync(BinaryData.FromString("Baz!")); // Set the access tier for all three blobs at once BlobBatchClient batch = service.GetBlobBatchClient(); await batch.SetBlobsAccessTierAsync(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool); } finally { // Clean up after the test when we're finished await container.DeleteAsync(); } }
public async Task BatchDeleteAsync() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container")); await container.CreateAsync(); try { // Create three blobs named "foo", "bar", and "baz" BlobClient foo = container.GetBlobClient("foo"); BlobClient bar = container.GetBlobClient("bar"); BlobClient baz = container.GetBlobClient("baz"); await foo.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Foo!"))); await bar.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Bar!"))); await baz.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Baz!"))); // Delete all three blobs at once BlobBatchClient batch = service.GetBlobBatchClient(); await batch.DeleteBlobsAsync(new Uri[] { foo.Uri, bar.Uri, baz.Uri }); } finally { // Clean up after the test when we're finished await container.DeleteAsync(); } }
public void BatchSetAccessTierCool() { string connectionString = ConnectionString; string containerName = Randomize("sample-container"); #region Snippet:SampleSnippetsBatch_AccessTier // Get a connection string to our Azure Storage account. //@@ string connectionString = "<connection_string>"; //@@ string containerName = "sample-container"; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(containerName); container.Create(); // Create three blobs named "foo", "bar", and "baz" BlobClient foo = container.GetBlobClient("foo"); BlobClient bar = container.GetBlobClient("bar"); BlobClient baz = container.GetBlobClient("baz"); foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!"))); bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!"))); baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!"))); // Set the access tier for all three blobs at once BlobBatchClient batch = service.GetBlobBatchClient(); batch.SetBlobsAccessTier(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool); #endregion foreach (BlobItem blob in container.GetBlobs()) { Assert.AreEqual(AccessTier.Cool, blob.Properties.AccessTier); } // Clean up after the test when we're finished }
public void BatchSetAccessTier() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container")); container.Create(); try { // Create three blobs named "foo", "bar", and "baz" BlobClient foo = container.GetBlobClient("foo"); BlobClient bar = container.GetBlobClient("bar"); BlobClient baz = container.GetBlobClient("baz"); foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!"))); bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!"))); baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!"))); // Set the access tier for all three blobs at once BlobBatchClient batch = service.GetBlobBatchClient(); batch.SetBlobsAccessTier(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool); } finally { // Clean up after the test when we're finished container.Delete(); } }
/// <summary> /// Authenticate with <see cref="DefaultAzureCredential"/>. /// </summary> public void Authenticate() { #region Snippet:SampleSnippetsBlobBatch_Auth // Create a BlobServiceClient that will authenticate through Active Directory Uri accountUri = new Uri("https://MYSTORAGEACCOUNT.blob.core.windows.net/"); BlobServiceClient client = new BlobServiceClient(accountUri, new DefaultAzureCredential()); BlobBatchClient batch = client.GetBlobBatchClient(); #endregion }
/// <summary> /// Call the batch API with a bunch of URLs to change the tier of /// </summary> /// <param name="blobServiceClient">connection to blob service</param> /// <param name="uris">urls to move</param> /// <param name="cancellationToken"></param> /// <returns></returns> private async Task ProcessBatch(BlobServiceClient blobServiceClient, IEnumerable <Uri> uris, CancellationToken cancellationToken) { _logger.LogInformation($"Sending Batch of {uris.Count()} items"); using (var op = _telemetryClient.StartOperation <DependencyTelemetry>("ProcessBatch")) { op.Telemetry.Properties.Add("Run", _config.Run); op.Telemetry.Metrics.Add("BatchSize", uris.Count()); if (!_config.WhatIf) { BlobBatchClient batch = blobServiceClient.GetBlobBatchClient(); await batch.SetBlobsAccessTierAsync(uris, _targetAccessTier, cancellationToken : cancellationToken); } } }
public async Task Delete_Error() { // Arrange BlobServiceClient service = GetServiceClient_SharedKey(); BlobServiceClient invalidServiceClient = InstrumentClient(new BlobServiceClient( GetServiceClient_SharedKey().Uri, GetOptions())); BlobBatchClient blobBatchClient = invalidServiceClient.GetBlobBatchClient(); using BlobBatch batch = blobBatchClient.CreateBatch(); batch.DeleteBlob(new Uri("https://account.blob.core.windows.net/container/blob")); // Act await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>( blobBatchClient.SubmitBatchAsync(batch), e => Assert.AreEqual(BlobErrorCode.NoAuthenticationInformation.ToString(), e.ErrorCode)); }
public async Task <bool> UpdateBlobStorageAccessSettingsArchive(string url) { try { BlobServiceClient cloudBlobServiceClient = new BlobServiceClient(_accessKey); // Set the access tier BlobBatchClient batch = cloudBlobServiceClient.GetBlobBatchClient(); await batch.SetBlobsAccessTierAsync(new Uri[] { new Uri(url) }, AccessTier.Archive); return(true); } catch { //add logging throw; } }
public async Task FineGrainedBatchingAsync() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container")); await container.CreateAsync(); try { // Create three blobs named "foo", "bar", and "baz" BlobClient foo = container.GetBlobClient("foo"); BlobClient bar = container.GetBlobClient("bar"); BlobClient baz = container.GetBlobClient("baz"); await foo.UploadAsync(BinaryData.FromString("Foo!")); await bar.UploadAsync(BinaryData.FromString("Bar!")); await baz.UploadAsync(BinaryData.FromString("Baz!")); // Create a batch with three deletes BlobBatchClient batchClient = service.GetBlobBatchClient(); BlobBatch batch = batchClient.CreateBatch(); Response fooResponse = batch.DeleteBlob(foo.Uri, DeleteSnapshotsOption.IncludeSnapshots); Response barResponse = batch.DeleteBlob(bar.Uri); Response bazResponse = batch.DeleteBlob(baz.Uri); // Submit the batch await batchClient.SubmitBatchAsync(batch); // Verify the results Assert.AreEqual(202, fooResponse.Status); Assert.AreEqual(202, barResponse.Status); Assert.AreEqual(202, bazResponse.Status); } finally { // Clean up after the test when we're finished await container.DeleteAsync(); } }
public void FineGrainedBatching() { string connectionString = ConnectionString; string containerName = Randomize("sample-container"); #region Snippet:SampleSnippetsBatch_FineGrainedBatching // Get a connection string to our Azure Storage account. //@@ string connectionString = "<connection_string>"; //@@ string containerName = "sample-container"; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(containerName); container.Create(); // Create three blobs named "foo", "bar", and "baz" BlobClient foo = container.GetBlobClient("foo"); BlobClient bar = container.GetBlobClient("bar"); BlobClient baz = container.GetBlobClient("baz"); foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!"))); foo.CreateSnapshot(); bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!"))); bar.CreateSnapshot(); baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!"))); // Create a batch with three deletes BlobBatchClient batchClient = service.GetBlobBatchClient(); BlobBatch batch = batchClient.CreateBatch(); batch.DeleteBlob(foo.Uri, DeleteSnapshotsOption.IncludeSnapshots); batch.DeleteBlob(bar.Uri, DeleteSnapshotsOption.OnlySnapshots); batch.DeleteBlob(baz.Uri); // Submit the batch batchClient.SubmitBatch(batch); #endregion Pageable <BlobItem> blobs = container.GetBlobs(states: BlobStates.Snapshots); Assert.AreEqual(1, blobs.Count()); Assert.AreEqual("bar", blobs.FirstOrDefault().Name); // Clean up after the test when we're finished container.Delete(); }
/* * Deletes all blobs in container */ public static void DeleteAllBlobsUsingBatch( BlobServiceClient serviceClient, BlobContainerClient containerClient) { // Getting list of blob uris Pageable <BlobItem> allBlobs = containerClient.GetBlobs(); Uri[] blobList = new Uri[allBlobs.Count()]; int count = 0; foreach (BlobItem blob in allBlobs) { BlobClient blobClient = containerClient.GetBlobClient(blob.Name); blobList[count] = blobClient.Uri; count++; } // Creating batch client then setting access tier for all blobs BlobBatchClient batchClient = serviceClient.GetBlobBatchClient(); batchClient.DeleteBlobs(blobList); }
private async Task ProcessBatch(BlobServiceClient blobServiceClient, IEnumerable <Uri> uris, CancellationToken cancellationToken) { _logger.LogInformation($"Sending Batch of {uris.Count()} items"); using (var op = _telemetryClient.StartOperation <DependencyTelemetry>("ProcessBatch")) { op.Telemetry.Properties.Add("Run", _config.Run); op.Telemetry.Metrics.Add("BatchSize", uris.Count()); if (!_config.WhatIf) { BlobBatchClient batch = blobServiceClient.GetBlobBatchClient(); if (_config.TargetAccessTier.Equals("Hot", StringComparison.InvariantCultureIgnoreCase)) { await batch.SetBlobsAccessTierAsync(uris, AccessTier.Hot, cancellationToken : cancellationToken); } else { await batch.SetBlobsAccessTierAsync(uris, AccessTier.Cool, cancellationToken : cancellationToken); } } } }
public void BatchErrors() { // Get a connection string to our Azure Storage account. string connectionString = ConnectionString; // Get a reference to a container named "sample-container" and then create it BlobServiceClient service = new BlobServiceClient(connectionString); BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container")); container.Create(); try { // Create a blob named "valid" BlobClient valid = container.GetBlobClient("valid"); valid.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Valid!"))); // Get a reference to a blob named "invalid", but never create it BlobClient invalid = container.GetBlobClient("invalid"); // Delete both blobs at the same time BlobBatchClient batch = service.GetBlobBatchClient(); batch.DeleteBlobs(new Uri[] { valid.Uri, invalid.Uri }); } catch (AggregateException ex) { // An aggregate exception is thrown for all the indivudal failures Assert.AreEqual(1, ex.InnerExceptions.Count); StorageRequestFailedException failure = ex.InnerException as StorageRequestFailedException; Assert.IsTrue(BlobErrorCode.BlobNotFound == failure.ErrorCode); } finally { // Clean up after the test when we're finished container.Delete(); } }
static int SasKeyExpiryTime = 1; // Default 1 hour static async Task Main(string[] args) { Console.WriteLine("Starting Blob operations"); if (String.IsNullOrEmpty(AccountKey)) { Console.WriteLine("The environment variable: AZURE_STORAGE_ACCOUNT_KEY, is not set!"); Environment.Exit(-1); } ; if (SourceTier.Equals(TargetTier)) { Console.WriteLine("Source and Target Access Tiers cannot be the same!"); Environment.Exit(-1); } ; Stopwatch stopWatch = new Stopwatch(); try { // Start the stop watch stopWatch.Start(); BlobServiceClient blobSvcClient = SharedAccessSignatureAuthAsync(); BlobContainerClient contClient = blobSvcClient.GetBlobContainerClient(ContainerName); Uri[] blobUrls = new Uri[BatchSize]; int batchCount = 0; int totalProcessed = 0; List <Task <Azure.Response[]> > taskList = new List <Task <Azure.Response[]> >(); BlobClient blobClient = null; IDictionary <string, string> dict = null; Console.WriteLine("Enumerating blobs ..."); await foreach (BlobItem blobItem in contClient.GetBlobsAsync(BlobTraits.Metadata)) { Console.WriteLine("-------------------------"); Console.WriteLine("\tName: " + blobItem.Name); blobClient = contClient.GetBlobClient(blobItem.Name); Console.WriteLine("\tUri: " + blobClient.Uri); BlobItemProperties props = blobItem.Properties; AccessTier?atier = props.AccessTier; Console.WriteLine("\tAccess Tier: " + atier); if (atier.Equals(SourceTier)) { blobUrls[batchCount] = blobClient.Uri; batchCount++; } ; dict = blobItem.Metadata; if ((dict != null) && (dict.Count > 0)) { foreach (KeyValuePair <string, string> item in dict) { Console.WriteLine("Key: {0}, Value: {1}", item.Key, item.Value); } } else { Console.WriteLine("\tNo metadata for this blob item"); } if (batchCount == BatchSize) { BlobBatchClient batch = blobSvcClient.GetBlobBatchClient(); taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier)); totalProcessed += batchCount; batchCount = 0; Console.WriteLine("-------------------------"); Console.WriteLine($"No. of Blobs moved to {TargetTier} tier: {totalProcessed}"); if (taskList.Count >= ConcurrentTasks) { // Wait for existing tasks to finish before proceeeding. // This is to avoid out of resources issue. Task.WaitAll(taskList.ToArray()); taskList.Clear(); } } ; } if (batchCount > 0) { BlobBatchClient batch = blobSvcClient.GetBlobBatchClient(); taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier)); totalProcessed += batchCount; } ; if (taskList.Count > 0) { Task.WaitAll(taskList.ToArray()); // Wait for all Tasks to finish! } // Stop the timer stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}", ts.Hours, ts.Minutes, ts.Seconds); Console.WriteLine("------------------------------------"); Console.WriteLine($"Moved {totalProcessed} Blobs from {SourceTier} to {TargetTier} tier."); Console.WriteLine($"Total Runtime: {elapsedTime}"); Console.WriteLine("------------------------------------"); } catch (Exception ex) { Console.WriteLine("Encountered exception: " + ex); } }
public static void Execute() { Console.WriteLine("\n Code demonstrating a Batch Delete scenario. \n"); Console.WriteLine("\n [Warning this will delete all blobs in your account - Kindly type Yes to proceed] \n"); // Take consent from the user (as it might get run by mistake from someone) string value = Console.ReadLine(); if (value == "Yes") { Console.WriteLine("\n Proceeding with emptying all your blob containers!"); } else { Console.WriteLine("\n Exiting !"); return; } // Set output path for debugging purposes Helper helper = new Helper(); helper.SetConsoleOutPutPath(helper.RedirectOutputToFile, ".\\BatchDelete.txt"); int segmentSize = 256; // Set up clients Random random = new Random(); BlobServiceClient blobServiceClient = new BlobServiceClient(helper.ConnectionString); BlobContainerClient container1 = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString()); BlobContainerClient container2 = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString()); BlobBatchClient batch = blobServiceClient.GetBlobBatchClient(); container1.CreateIfNotExists(); container2.CreateIfNotExists(); Console.WriteLine("\n Creating some sample blobs for deletion."); // Create 50 blobs in both the containers for (int i = 0; i < 50; i++) { Console.WriteLine("Creating blob id = {0}", i.ToString()); BlobClient blob1 = container1.GetBlobClient("blob_" + i); blob1.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!"))); BlobClient blob2 = container2.GetBlobClient("blob_" + i); blob2.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!"))); } // Call the listing operation and enumerate the result segment. var containerListResult = blobServiceClient.GetBlobContainers() .AsPages(null, segmentSize); foreach (var containerPage in containerListResult) { foreach (var containerItem in containerPage.Values) { Console.WriteLine("Container name: {0}", containerItem.Name); // Call the listing operation and return pages of the specified size. var containerClient = blobServiceClient.GetBlobContainerClient(containerItem.Name); var resultSegment = containerClient.GetBlobs() .AsPages(null, segmentSize); // Enumerate the blobs returned for each page. foreach (var blobPage in resultSegment) { List <Uri> Urilist = new List <Uri>(); foreach (var blobItem in blobPage.Values) { Console.WriteLine("Adding Blob to delete queue: {0}", blobItem.Name); Urilist.Add(containerClient.GetBlobClient(blobItem.Name).Uri); } // Delete blobs at once try { if (Urilist.Count > 0) { var response = batch.DeleteBlobs(Urilist); } } catch (AggregateException ex) { Console.WriteLine(ex.Message.ToString()); } } } } Console.WriteLine("All the blobs in the account are deleted successfully."); }