public void BatchErrors()
{
string connectionString = ConnectionString;
string containerName = Randomize("sample-container");
#region Snippet:SampleSnippetsBatch_Troubleshooting
// Get a connection string to our Azure Storage account.
//@@ string connectionString = "<connection_string>";
//@@ string containerName = "sample-container";
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(containerName);
container.Create();
// Create a blob named "valid"
BlobClient valid = container.GetBlobClient("valid");
valid.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Valid!")));
// Get a reference to a blob named "invalid", but never create it
BlobClient invalid = container.GetBlobClient("invalid");
// Delete both blobs at the same time
BlobBatchClient batch = service.GetBlobBatchClient();
try
{
batch.DeleteBlobs(new Uri[] { valid.Uri, invalid.Uri });
}
catch (AggregateException)
{
// An aggregate exception is thrown for all the individual failures
// Check ex.InnerExceptions for RequestFailedException instances
}
#endregion
}
public void BatchDelete()
{
string connectionString = ConnectionString;
string containerName = Randomize("sample-container");
#region Snippet:SampleSnippetsBatch_DeleteBatch
// Get a connection string to our Azure Storage account.
//@@ string connectionString = "<connection_string>";
//@@ string containerName = "sample-container";
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(containerName);
container.Create();
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));
// Delete all three blobs at once
BlobBatchClient batch = service.GetBlobBatchClient();
batch.DeleteBlobs(new Uri[] { foo.Uri, bar.Uri, baz.Uri });
#endregion
Assert.AreEqual(0, container.GetBlobs().ToList().Count);
// Clean up after we're finished
container.Delete();
}
public async Task BatchSetAccessTierAsync()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
await container.CreateAsync();
try
{
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
await foo.UploadAsync(BinaryData.FromString("Foo!"));
await bar.UploadAsync(BinaryData.FromString("Bar!"));
await baz.UploadAsync(BinaryData.FromString("Baz!"));
// Set the access tier for all three blobs at once
BlobBatchClient batch = service.GetBlobBatchClient();
await batch.SetBlobsAccessTierAsync(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool);
}
finally
{
// Clean up after the test when we're finished
await container.DeleteAsync();
}
}
public async Task BatchDeleteAsync()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
await container.CreateAsync();
try
{
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
await foo.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
await bar.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
await baz.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));
// Delete all three blobs at once
BlobBatchClient batch = service.GetBlobBatchClient();
await batch.DeleteBlobsAsync(new Uri[] { foo.Uri, bar.Uri, baz.Uri });
}
finally
{
// Clean up after the test when we're finished
await container.DeleteAsync();
}
}
public void BatchSetAccessTierCool()
{
string connectionString = ConnectionString;
string containerName = Randomize("sample-container");
#region Snippet:SampleSnippetsBatch_AccessTier
// Get a connection string to our Azure Storage account.
//@@ string connectionString = "<connection_string>";
//@@ string containerName = "sample-container";
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(containerName);
container.Create();
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));
// Set the access tier for all three blobs at once
BlobBatchClient batch = service.GetBlobBatchClient();
batch.SetBlobsAccessTier(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool);
#endregion
foreach (BlobItem blob in container.GetBlobs())
{
Assert.AreEqual(AccessTier.Cool, blob.Properties.AccessTier);
}
// Clean up after the test when we're finished
}
public void BatchSetAccessTier()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
container.Create();
try
{
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));
// Set the access tier for all three blobs at once
BlobBatchClient batch = service.GetBlobBatchClient();
batch.SetBlobsAccessTier(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool);
}
finally
{
// Clean up after the test when we're finished
container.Delete();
}
}
/// <summary>
/// Authenticate with <see cref="DefaultAzureCredential"/>.
/// </summary>
public void Authenticate()
{
#region Snippet:SampleSnippetsBlobBatch_Auth
// Create a BlobServiceClient that will authenticate through Active Directory
Uri accountUri = new Uri("https://MYSTORAGEACCOUNT.blob.core.windows.net/");
BlobServiceClient client = new BlobServiceClient(accountUri, new DefaultAzureCredential());
BlobBatchClient batch = client.GetBlobBatchClient();
#endregion
}
/// <summary>
/// Call the batch API with a bunch of URLs to change the tier of
/// </summary>
/// <param name="blobServiceClient">connection to blob service</param>
/// <param name="uris">urls to move</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task ProcessBatch(BlobServiceClient blobServiceClient, IEnumerable <Uri> uris, CancellationToken cancellationToken)
{
_logger.LogInformation($"Sending Batch of {uris.Count()} items");
using (var op = _telemetryClient.StartOperation <DependencyTelemetry>("ProcessBatch"))
{
op.Telemetry.Properties.Add("Run", _config.Run);
op.Telemetry.Metrics.Add("BatchSize", uris.Count());
if (!_config.WhatIf)
{
BlobBatchClient batch = blobServiceClient.GetBlobBatchClient();
await batch.SetBlobsAccessTierAsync(uris, _targetAccessTier, cancellationToken : cancellationToken);
}
}
}
public async Task Delete_Error()
{
// Arrange
BlobServiceClient service = GetServiceClient_SharedKey();
BlobServiceClient invalidServiceClient = InstrumentClient(new BlobServiceClient(
GetServiceClient_SharedKey().Uri,
GetOptions()));
BlobBatchClient blobBatchClient = invalidServiceClient.GetBlobBatchClient();
using BlobBatch batch = blobBatchClient.CreateBatch();
batch.DeleteBlob(new Uri("https://account.blob.core.windows.net/container/blob"));
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blobBatchClient.SubmitBatchAsync(batch),
e => Assert.AreEqual(BlobErrorCode.NoAuthenticationInformation.ToString(), e.ErrorCode));
}
public async Task <bool> UpdateBlobStorageAccessSettingsArchive(string url)
{
try
{
BlobServiceClient cloudBlobServiceClient = new BlobServiceClient(_accessKey);
// Set the access tier
BlobBatchClient batch = cloudBlobServiceClient.GetBlobBatchClient();
await batch.SetBlobsAccessTierAsync(new Uri[] { new Uri(url) }, AccessTier.Archive);
return(true);
}
catch
{
//add logging
throw;
}
}
public async Task FineGrainedBatchingAsync()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
await container.CreateAsync();
try
{
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
await foo.UploadAsync(BinaryData.FromString("Foo!"));
await bar.UploadAsync(BinaryData.FromString("Bar!"));
await baz.UploadAsync(BinaryData.FromString("Baz!"));
// Create a batch with three deletes
BlobBatchClient batchClient = service.GetBlobBatchClient();
BlobBatch batch = batchClient.CreateBatch();
Response fooResponse = batch.DeleteBlob(foo.Uri, DeleteSnapshotsOption.IncludeSnapshots);
Response barResponse = batch.DeleteBlob(bar.Uri);
Response bazResponse = batch.DeleteBlob(baz.Uri);
// Submit the batch
await batchClient.SubmitBatchAsync(batch);
// Verify the results
Assert.AreEqual(202, fooResponse.Status);
Assert.AreEqual(202, barResponse.Status);
Assert.AreEqual(202, bazResponse.Status);
}
finally
{
// Clean up after the test when we're finished
await container.DeleteAsync();
}
}
public void FineGrainedBatching()
{
string connectionString = ConnectionString;
string containerName = Randomize("sample-container");
#region Snippet:SampleSnippetsBatch_FineGrainedBatching
// Get a connection string to our Azure Storage account.
//@@ string connectionString = "<connection_string>";
//@@ string containerName = "sample-container";
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(containerName);
container.Create();
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
foo.CreateSnapshot();
bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
bar.CreateSnapshot();
baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));
// Create a batch with three deletes
BlobBatchClient batchClient = service.GetBlobBatchClient();
BlobBatch batch = batchClient.CreateBatch();
batch.DeleteBlob(foo.Uri, DeleteSnapshotsOption.IncludeSnapshots);
batch.DeleteBlob(bar.Uri, DeleteSnapshotsOption.OnlySnapshots);
batch.DeleteBlob(baz.Uri);
// Submit the batch
batchClient.SubmitBatch(batch);
#endregion
Pageable <BlobItem> blobs = container.GetBlobs(states: BlobStates.Snapshots);
Assert.AreEqual(1, blobs.Count());
Assert.AreEqual("bar", blobs.FirstOrDefault().Name);
// Clean up after the test when we're finished
container.Delete();
}
/*
* Deletes all blobs in container
*/
public static void DeleteAllBlobsUsingBatch(
BlobServiceClient serviceClient,
BlobContainerClient containerClient)
{
// Getting list of blob uris
Pageable <BlobItem> allBlobs = containerClient.GetBlobs();
Uri[] blobList = new Uri[allBlobs.Count()];
int count = 0;
foreach (BlobItem blob in allBlobs)
{
BlobClient blobClient = containerClient.GetBlobClient(blob.Name);
blobList[count] = blobClient.Uri;
count++;
}
// Creating batch client then setting access tier for all blobs
BlobBatchClient batchClient = serviceClient.GetBlobBatchClient();
batchClient.DeleteBlobs(blobList);
}
private async Task ProcessBatch(BlobServiceClient blobServiceClient, IEnumerable <Uri> uris, CancellationToken cancellationToken)
{
_logger.LogInformation($"Sending Batch of {uris.Count()} items");
using (var op = _telemetryClient.StartOperation <DependencyTelemetry>("ProcessBatch"))
{
op.Telemetry.Properties.Add("Run", _config.Run);
op.Telemetry.Metrics.Add("BatchSize", uris.Count());
if (!_config.WhatIf)
{
BlobBatchClient batch = blobServiceClient.GetBlobBatchClient();
if (_config.TargetAccessTier.Equals("Hot", StringComparison.InvariantCultureIgnoreCase))
{
await batch.SetBlobsAccessTierAsync(uris, AccessTier.Hot, cancellationToken : cancellationToken);
}
else
{
await batch.SetBlobsAccessTierAsync(uris, AccessTier.Cool, cancellationToken : cancellationToken);
}
}
}
}
public void BatchErrors()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
container.Create();
try
{
// Create a blob named "valid"
BlobClient valid = container.GetBlobClient("valid");
valid.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Valid!")));
// Get a reference to a blob named "invalid", but never create it
BlobClient invalid = container.GetBlobClient("invalid");
// Delete both blobs at the same time
BlobBatchClient batch = service.GetBlobBatchClient();
batch.DeleteBlobs(new Uri[] { valid.Uri, invalid.Uri });
}
catch (AggregateException ex)
{
// An aggregate exception is thrown for all the indivudal failures
Assert.AreEqual(1, ex.InnerExceptions.Count);
StorageRequestFailedException failure = ex.InnerException as StorageRequestFailedException;
Assert.IsTrue(BlobErrorCode.BlobNotFound == failure.ErrorCode);
}
finally
{
// Clean up after the test when we're finished
container.Delete();
}
}
static int SasKeyExpiryTime = 1; // Default 1 hour
static async Task Main(string[] args)
{
Console.WriteLine("Starting Blob operations");
if (String.IsNullOrEmpty(AccountKey))
{
Console.WriteLine("The environment variable: AZURE_STORAGE_ACCOUNT_KEY, is not set!");
Environment.Exit(-1);
}
;
if (SourceTier.Equals(TargetTier))
{
Console.WriteLine("Source and Target Access Tiers cannot be the same!");
Environment.Exit(-1);
}
;
Stopwatch stopWatch = new Stopwatch();
try
{
// Start the stop watch
stopWatch.Start();
BlobServiceClient blobSvcClient = SharedAccessSignatureAuthAsync();
BlobContainerClient contClient = blobSvcClient.GetBlobContainerClient(ContainerName);
Uri[] blobUrls = new Uri[BatchSize];
int batchCount = 0;
int totalProcessed = 0;
List <Task <Azure.Response[]> > taskList = new List <Task <Azure.Response[]> >();
BlobClient blobClient = null;
IDictionary <string, string> dict = null;
Console.WriteLine("Enumerating blobs ...");
await foreach (BlobItem blobItem in contClient.GetBlobsAsync(BlobTraits.Metadata))
{
Console.WriteLine("-------------------------");
Console.WriteLine("\tName: " + blobItem.Name);
blobClient = contClient.GetBlobClient(blobItem.Name);
Console.WriteLine("\tUri: " + blobClient.Uri);
BlobItemProperties props = blobItem.Properties;
AccessTier?atier = props.AccessTier;
Console.WriteLine("\tAccess Tier: " + atier);
if (atier.Equals(SourceTier))
{
blobUrls[batchCount] = blobClient.Uri;
batchCount++;
}
;
dict = blobItem.Metadata;
if ((dict != null) && (dict.Count > 0))
{
foreach (KeyValuePair <string, string> item in dict)
{
Console.WriteLine("Key: {0}, Value: {1}", item.Key, item.Value);
}
}
else
{
Console.WriteLine("\tNo metadata for this blob item");
}
if (batchCount == BatchSize)
{
BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
totalProcessed += batchCount;
batchCount = 0;
Console.WriteLine("-------------------------");
Console.WriteLine($"No. of Blobs moved to {TargetTier} tier: {totalProcessed}");
if (taskList.Count >= ConcurrentTasks)
{
// Wait for existing tasks to finish before proceeeding.
// This is to avoid out of resources issue.
Task.WaitAll(taskList.ToArray());
taskList.Clear();
}
}
;
}
if (batchCount > 0)
{
BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
totalProcessed += batchCount;
}
;
if (taskList.Count > 0)
{
Task.WaitAll(taskList.ToArray()); // Wait for all Tasks to finish!
}
// Stop the timer
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}",
ts.Hours, ts.Minutes, ts.Seconds);
Console.WriteLine("------------------------------------");
Console.WriteLine($"Moved {totalProcessed} Blobs from {SourceTier} to {TargetTier} tier.");
Console.WriteLine($"Total Runtime: {elapsedTime}");
Console.WriteLine("------------------------------------");
}
catch (Exception ex)
{
Console.WriteLine("Encountered exception: " + ex);
}
}
public static void Execute()
{
Console.WriteLine("\n Code demonstrating a Batch Delete scenario. \n");
Console.WriteLine("\n [Warning this will delete all blobs in your account - Kindly type Yes to proceed] \n");
// Take consent from the user (as it might get run by mistake from someone)
string value = Console.ReadLine();
if (value == "Yes")
{
Console.WriteLine("\n Proceeding with emptying all your blob containers!");
}
else
{
Console.WriteLine("\n Exiting !");
return;
}
// Set output path for debugging purposes
Helper helper = new Helper();
helper.SetConsoleOutPutPath(helper.RedirectOutputToFile, ".\\BatchDelete.txt");
int segmentSize = 256;
// Set up clients
Random random = new Random();
BlobServiceClient blobServiceClient = new BlobServiceClient(helper.ConnectionString);
BlobContainerClient container1 = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString());
BlobContainerClient container2 = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString());
BlobBatchClient batch = blobServiceClient.GetBlobBatchClient();
container1.CreateIfNotExists();
container2.CreateIfNotExists();
Console.WriteLine("\n Creating some sample blobs for deletion.");
// Create 50 blobs in both the containers
for (int i = 0; i < 50; i++)
{
Console.WriteLine("Creating blob id = {0}", i.ToString());
BlobClient blob1 = container1.GetBlobClient("blob_" + i);
blob1.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!")));
BlobClient blob2 = container2.GetBlobClient("blob_" + i);
blob2.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!")));
}
// Call the listing operation and enumerate the result segment.
var containerListResult =
blobServiceClient.GetBlobContainers()
.AsPages(null, segmentSize);
foreach (var containerPage in containerListResult)
{
foreach (var containerItem in containerPage.Values)
{
Console.WriteLine("Container name: {0}", containerItem.Name);
// Call the listing operation and return pages of the specified size.
var containerClient = blobServiceClient.GetBlobContainerClient(containerItem.Name);
var resultSegment = containerClient.GetBlobs()
.AsPages(null, segmentSize);
// Enumerate the blobs returned for each page.
foreach (var blobPage in resultSegment)
{
List <Uri> Urilist = new List <Uri>();
foreach (var blobItem in blobPage.Values)
{
Console.WriteLine("Adding Blob to delete queue: {0}", blobItem.Name);
Urilist.Add(containerClient.GetBlobClient(blobItem.Name).Uri);
}
// Delete blobs at once
try
{
if (Urilist.Count > 0)
{
var response = batch.DeleteBlobs(Urilist);
}
}
catch (AggregateException ex)
{
Console.WriteLine(ex.Message.ToString());
}
}
}
}
Console.WriteLine("All the blobs in the account are deleted successfully.");
}
