public async Task Delete_OneFails_Convenience()
{
await using TestScenario scenario = Scenario();
BlobClient[] good = await scenario.CreateBlobsAsync(2);
Uri[] bad = scenario.GetInvalidBlobUris(1);
Uri[] uris = good.Select(b => b.Uri).Concat(bad).ToArray();
BlobBatchClient client = scenario.GetBlobBatchClient();
AggregateException exes = Assert.ThrowsAsync <AggregateException>(
async() => await client.DeleteBlobsAsync(uris));
RequestFailedException ex = exes.InnerException as RequestFailedException;
Assert.IsNotNull(ex);
Assert.AreEqual(404, ex.Status);
Assert.IsTrue(BlobErrorCode.ContainerNotFound == ex.ErrorCode);
await scenario.AssertDeleted(good);
}
public async Task Delete_OneFails_NoThrow()
{
await using TestScenario scenario = Scenario();
BlobClient[] good = await scenario.CreateBlobsAsync(2);
Uri[] bad = scenario.GetInvalidBlobUris(1);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response response1 = batch.DeleteBlob(good[0].Uri);
Response response2 = batch.DeleteBlob(good[1].Uri);
Response response3 = batch.DeleteBlob(bad[0]);
Response response = await client.SubmitBatchAsync(batch, throwOnAnyFailure : false);
Assert.AreEqual(3, batch.RequestCount);
scenario.AssertStatus(202, response, response1, response2);
scenario.AssertStatus(404, response3);
await scenario.AssertDeleted(good);
}
public async Task Delete_Basic()
{
await using TestScenario scenario = Scenario();
BlobClient[] blobs = await scenario.CreateBlobsAsync(3);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response[] responses = new Response[]
{
batch.DeleteBlob(blobs[0].Uri),
batch.DeleteBlob(blobs[1].Uri),
batch.DeleteBlob(blobs[2].Uri)
};
Response response = await client.SubmitBatchAsync(batch);
scenario.AssertStatus(202, response);
scenario.AssertStatus(202, responses);
await scenario.AssertDeleted(blobs);
}
public async Task SetBlobAccessTier_MultipleFail_NoThrow()
{
await using TestScenario scenario = Scenario();
BlobClient[] good = await scenario.CreateBlobsAsync(1);
Uri[] bad = scenario.GetInvalidBlobUris(2);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response response1 = batch.SetBlobAccessTier(good[0].Uri, AccessTier.Cool);
Response response2 = batch.SetBlobAccessTier(bad[0], AccessTier.Cool);
Response response3 = batch.SetBlobAccessTier(bad[1], AccessTier.Cool);
Response response = await client.SubmitBatchAsync(batch, throwOnAnyFailure : false);
scenario.AssertStatus(202, response);
scenario.AssertStatus(200, response1);
scenario.AssertStatus(404, response2, response3);
await scenario.AssertTiers(AccessTier.Cool, good);
}
public async Task SetBlobAccessTier_MultipleFail()
{
using TestScenario scenario = Scenario();
BlobClient[] good = await scenario.CreateBlobsAsync(1);
Uri[] bad = scenario.GetInvalidBlobUris(2);
BlobBatchClient client = scenario.GetBlobBatchClient();
BlobBatch batch = client.CreateBatch();
Response response1 = batch.SetBlobAccessTier(good[0].Uri, AccessTier.Cool);
Response response2 = batch.SetBlobAccessTier(bad[0], AccessTier.Cool);
Response response3 = batch.SetBlobAccessTier(bad[1], AccessTier.Cool);
AggregateException exes = Assert.ThrowsAsync <AggregateException>(
async() => await client.SubmitBatchAsync(batch));
Assert.AreEqual(2, exes.InnerExceptions.Count);
Assert.AreEqual(404, (exes.InnerExceptions[0] as StorageRequestFailedException)?.Status);
Assert.AreEqual(404, (exes.InnerExceptions[1] as StorageRequestFailedException)?.Status);
await scenario.AssertTiers(AccessTier.Cool, good);
}
public async Task SetBlobAccessTier_Basic()
{
await using TestScenario scenario = Scenario();
BlobClient[] blobs = await scenario.CreateBlobsAsync(3);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response[] responses = new Response[]
{
batch.SetBlobAccessTier(blobs[0].Uri, AccessTier.Cool),
batch.SetBlobAccessTier(blobs[1].Uri, AccessTier.Cool),
batch.SetBlobAccessTier(blobs[2].Uri, AccessTier.Cool)
};
Response response = await client.SubmitBatchAsync(batch);
scenario.AssertStatus(202, response);
scenario.AssertStatus(200, responses);
await scenario.AssertTiers(AccessTier.Cool, blobs);
}
public void FineGrainedBatching()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
container.Create();
try
{
// Create three blobs named "foo", "bar", and "baz"
BlobClient foo = container.GetBlobClient("foo");
BlobClient bar = container.GetBlobClient("bar");
BlobClient baz = container.GetBlobClient("baz");
foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));
// Create a batch with three deletes
BlobBatchClient batchClient = service.GetBlobBatchClient();
BlobBatch batch = batchClient.CreateBatch();
Response fooResponse = batch.DeleteBlob(foo.Uri, DeleteSnapshotsOption.Include);
Response barResponse = batch.DeleteBlob(bar.Uri);
Response bazResponse = batch.DeleteBlob(baz.Uri);
// Submit the batch
batchClient.SubmitBatch(batch);
// Verify the results
Assert.AreEqual(202, fooResponse.Status);
Assert.AreEqual(202, barResponse.Status);
Assert.AreEqual(202, bazResponse.Status);
}
finally
{
// Clean up after the test when we're finished
container.Delete();
}
}
public async Task Batch_Dispose_Response_Still_Available()
{
await using TestScenario scenario = Scenario();
BlobClient[] blobs = await scenario.CreateBlobsAsync(3);
BlobBatchClient client = scenario.GetBlobBatchClient();
Response[] responses = new Response[3];
Response response;
using (BlobBatch batch = client.CreateBatch())
{
responses[0] = batch.DeleteBlob(blobs[0].Uri);
responses[1] = batch.DeleteBlob(blobs[1].Uri);
responses[2] = batch.DeleteBlob(blobs[2].Uri);
response = await client.SubmitBatchAsync(batch);
}
scenario.AssertStatus(202, response);
scenario.AssertStatus(202, responses);
await scenario.AssertDeleted(blobs);
}
public async Task Delete_Error()
{
// Arrange
BlobServiceClient service = GetServiceClient_SharedKey();
BlobServiceClient invalidServiceClient = InstrumentClient(new BlobServiceClient(
GetServiceClient_SharedKey().Uri,
GetOptions()));
BlobBatchClient blobBatchClient = invalidServiceClient.GetBlobBatchClient();
using BlobBatch batch = blobBatchClient.CreateBatch();
batch.DeleteBlob(new Uri("https://account.blob.core.windows.net/container/blob"));
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
blobBatchClient.SubmitBatchAsync(batch),
e => Assert.AreEqual(
_serviceVersion >= BlobClientOptions.ServiceVersion.V2019_12_12 ?
BlobErrorCode.NoAuthenticationInformation.ToString() :
BlobErrorCode.AuthenticationFailed.ToString(),
e.ErrorCode));
}
public async Task Delete_MultipleFail()
{
await using TestScenario scenario = Scenario();
BlobClient[] good = await scenario.CreateBlobsAsync(1);
Uri[] bad = scenario.GetInvalidBlobUris(2);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response response1 = batch.DeleteBlob(good[0].Uri);
Response response2 = batch.DeleteBlob(bad[0]);
Response response3 = batch.DeleteBlob(bad[1]);
AggregateException exes = Assert.ThrowsAsync <AggregateException>(
async() => await client.SubmitBatchAsync(batch, throwOnAnyFailure: true));
Assert.AreEqual(2, exes.InnerExceptions.Count);
Assert.AreEqual(404, (exes.InnerExceptions[0] as RequestFailedException)?.Status);
Assert.AreEqual(404, (exes.InnerExceptions[1] as RequestFailedException)?.Status);
await scenario.AssertDeleted(good);
}
public async Task SetBlobAccessTier_Version()
{
await using TestScenario scenario = Scenario();
BlockBlobClient[] blobs = await scenario.CreateBlockBlobsAsync(1);
Response <BlobInfo> setMetadataResponse = await blobs[0].SetMetadataAsync(BuildMetadata());
blobs[0] = blobs[0].WithVersion(setMetadataResponse.Value.VersionId);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response[] responses = new Response[]
{
batch.SetBlobAccessTier(blobs[0].Uri, AccessTier.Cool),
};
Response response = await client.SubmitBatchAsync(batch);
scenario.AssertStatus(202, response);
scenario.AssertStatus(200, responses);
await scenario.AssertTiers(AccessTier.Cool, blobs);
}
/*
* Deletes all blobs in container
*/
public static void DeleteAllBlobsUsingBatch(
BlobServiceClient serviceClient,
BlobContainerClient containerClient)
{
// Getting list of blob uris
Pageable <BlobItem> allBlobs = containerClient.GetBlobs();
Uri[] blobList = new Uri[allBlobs.Count()];
int count = 0;
foreach (BlobItem blob in allBlobs)
{
BlobClient blobClient = containerClient.GetBlobClient(blob.Name);
blobList[count] = blobClient.Uri;
count++;
}
// Creating batch client then setting access tier for all blobs
BlobBatchClient batchClient = serviceClient.GetBlobBatchClient();
batchClient.DeleteBlobs(blobList);
}
public async Task SetBlobAccessTier_OneFails()
{
using TestScenario scenario = Scenario();
BlobClient[] good = await scenario.CreateBlobsAsync(2);
Uri[] bad = scenario.GetInvalidBlobUris(1);
BlobBatchClient client = scenario.GetBlobBatchClient();
BlobBatch batch = client.CreateBatch();
Response response1 = batch.SetBlobAccessTier(good[0].Uri, AccessTier.Cool);
Response response2 = batch.SetBlobAccessTier(good[1].Uri, AccessTier.Cool);
Response response3 = batch.SetBlobAccessTier(bad[0], AccessTier.Cool);
AggregateException exes = Assert.ThrowsAsync <AggregateException>(
async() => await client.SubmitBatchAsync(batch));
RequestFailedException ex = exes.InnerException as RequestFailedException;
Assert.IsNotNull(ex);
Assert.AreEqual(404, ex.Status);
Assert.IsTrue(BlobErrorCode.ContainerNotFound == ex.ErrorCode);
await scenario.AssertTiers(AccessTier.Cool, good);
}
public async Task SetBlobAccessTier_Snapshot()
{
await using TestScenario scenario = Scenario();
BlockBlobClient[] blobs = await scenario.CreateBlockBlobsAsync(1);
Response <BlobSnapshotInfo> blobSnapshotResponse = await blobs[0].CreateSnapshotAsync();
blobs[0] = blobs[0].WithSnapshot(blobSnapshotResponse.Value.Snapshot);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response[] responses = new Response[]
{
batch.SetBlobAccessTier(blobs[0].Uri, AccessTier.Cool),
};
Response response = await client.SubmitBatchAsync(batch);
scenario.AssertStatus(202, response);
scenario.AssertStatus(200, responses);
await scenario.AssertTiers(AccessTier.Cool, blobs);
}
public async Task Delete_OneFails()
{
await using TestScenario scenario = Scenario();
BlobClient[] good = await scenario.CreateBlobsAsync(2);
Uri[] bad = scenario.GetInvalidBlobUris(1);
BlobBatchClient client = scenario.GetBlobBatchClient();
using BlobBatch batch = client.CreateBatch();
Response response1 = batch.DeleteBlob(good[0].Uri);
Response response2 = batch.DeleteBlob(good[1].Uri);
Response response3 = batch.DeleteBlob(bad[0]);
AggregateException exes = Assert.ThrowsAsync <AggregateException>(
async() => await client.SubmitBatchAsync(batch, throwOnAnyFailure: true));
RequestFailedException ex = exes.InnerException as RequestFailedException;
Assert.IsNotNull(ex);
Assert.AreEqual(404, ex.Status);
Assert.IsTrue(BlobErrorCode.ContainerNotFound == ex.ErrorCode);
await scenario.AssertDeleted(good);
}
private async Task ProcessBatch(BlobServiceClient blobServiceClient, IEnumerable <Uri> uris, CancellationToken cancellationToken)
{
_logger.LogInformation($"Sending Batch of {uris.Count()} items");
using (var op = _telemetryClient.StartOperation <DependencyTelemetry>("ProcessBatch"))
{
op.Telemetry.Properties.Add("Run", _config.Run);
op.Telemetry.Metrics.Add("BatchSize", uris.Count());
if (!_config.WhatIf)
{
BlobBatchClient batch = blobServiceClient.GetBlobBatchClient();
if (_config.TargetAccessTier.Equals("Hot", StringComparison.InvariantCultureIgnoreCase))
{
await batch.SetBlobsAccessTierAsync(uris, AccessTier.Hot, cancellationToken : cancellationToken);
}
else
{
await batch.SetBlobsAccessTierAsync(uris, AccessTier.Cool, cancellationToken : cancellationToken);
}
}
}
}
public async Task Batch_CanUseResponseAfterException()
{
using TestScenario scenario = Scenario();
Uri[] good = await scenario.CreateBlobUrisAsync(1);
Uri[] bad = scenario.GetInvalidBlobUris(1);
BlobBatchClient client = scenario.GetBlobBatchClient();
BlobBatch batch = client.CreateBatch();
Response response1 = batch.DeleteBlob(good[0]);
Response response2 = batch.DeleteBlob(bad[0]);
try
{
await client.SubmitBatchAsync(batch);
}
catch (AggregateException)
{
// Swallow the exception
}
scenario.AssertStatus(202, response1);
scenario.AssertStatus(404, response2);
}
public void BatchErrors()
{
// Get a connection string to our Azure Storage account.
string connectionString = ConnectionString;
// Get a reference to a container named "sample-container" and then create it
BlobServiceClient service = new BlobServiceClient(connectionString);
BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
container.Create();
try
{
// Create a blob named "valid"
BlobClient valid = container.GetBlobClient("valid");
valid.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Valid!")));
// Get a reference to a blob named "invalid", but never create it
BlobClient invalid = container.GetBlobClient("invalid");
// Delete both blobs at the same time
BlobBatchClient batch = service.GetBlobBatchClient();
batch.DeleteBlobs(new Uri[] { valid.Uri, invalid.Uri });
}
catch (AggregateException ex)
{
// An aggregate exception is thrown for all the indivudal failures
Assert.AreEqual(1, ex.InnerExceptions.Count);
StorageRequestFailedException failure = ex.InnerException as StorageRequestFailedException;
Assert.IsTrue(BlobErrorCode.BlobNotFound == failure.ErrorCode);
}
finally
{
// Clean up after the test when we're finished
container.Delete();
}
}
public static void Execute()
{
Console.WriteLine("\n Code demonstrating a Batch Delete scenario. \n");
Console.WriteLine("\n [Warning this will delete all blobs in your account - Kindly type Yes to proceed] \n");
// Take consent from the user (as it might get run by mistake from someone)
string value = Console.ReadLine();
if (value == "Yes")
{
Console.WriteLine("\n Proceeding with emptying all your blob containers!");
}
else
{
Console.WriteLine("\n Exiting !");
return;
}
// Set output path for debugging purposes
Helper helper = new Helper();
helper.SetConsoleOutPutPath(helper.RedirectOutputToFile, ".\\BatchDelete.txt");
int segmentSize = 256;
// Set up clients
Random random = new Random();
BlobServiceClient blobServiceClient = new BlobServiceClient(helper.ConnectionString);
BlobContainerClient container1 = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString());
BlobContainerClient container2 = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString());
BlobBatchClient batch = blobServiceClient.GetBlobBatchClient();
container1.CreateIfNotExists();
container2.CreateIfNotExists();
Console.WriteLine("\n Creating some sample blobs for deletion.");
// Create 50 blobs in both the containers
for (int i = 0; i < 50; i++)
{
Console.WriteLine("Creating blob id = {0}", i.ToString());
BlobClient blob1 = container1.GetBlobClient("blob_" + i);
blob1.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!")));
BlobClient blob2 = container2.GetBlobClient("blob_" + i);
blob2.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!")));
}
// Call the listing operation and enumerate the result segment.
var containerListResult =
blobServiceClient.GetBlobContainers()
.AsPages(null, segmentSize);
foreach (var containerPage in containerListResult)
{
foreach (var containerItem in containerPage.Values)
{
Console.WriteLine("Container name: {0}", containerItem.Name);
// Call the listing operation and return pages of the specified size.
var containerClient = blobServiceClient.GetBlobContainerClient(containerItem.Name);
var resultSegment = containerClient.GetBlobs()
.AsPages(null, segmentSize);
// Enumerate the blobs returned for each page.
foreach (var blobPage in resultSegment)
{
List <Uri> Urilist = new List <Uri>();
foreach (var blobItem in blobPage.Values)
{
Console.WriteLine("Adding Blob to delete queue: {0}", blobItem.Name);
Urilist.Add(containerClient.GetBlobClient(blobItem.Name).Uri);
}
// Delete blobs at once
try
{
if (Urilist.Count > 0)
{
var response = batch.DeleteBlobs(Urilist);
}
}
catch (AggregateException ex)
{
Console.WriteLine(ex.Message.ToString());
}
}
}
}
Console.WriteLine("All the blobs in the account are deleted successfully.");
}
static int SasKeyExpiryTime = 1; // Default 1 hour
static async Task Main(string[] args)
{
Console.WriteLine("Starting Blob operations");
if (String.IsNullOrEmpty(AccountKey))
{
Console.WriteLine("The environment variable: AZURE_STORAGE_ACCOUNT_KEY, is not set!");
Environment.Exit(-1);
}
;
if (SourceTier.Equals(TargetTier))
{
Console.WriteLine("Source and Target Access Tiers cannot be the same!");
Environment.Exit(-1);
}
;
Stopwatch stopWatch = new Stopwatch();
try
{
// Start the stop watch
stopWatch.Start();
BlobServiceClient blobSvcClient = SharedAccessSignatureAuthAsync();
BlobContainerClient contClient = blobSvcClient.GetBlobContainerClient(ContainerName);
Uri[] blobUrls = new Uri[BatchSize];
int batchCount = 0;
int totalProcessed = 0;
List <Task <Azure.Response[]> > taskList = new List <Task <Azure.Response[]> >();
BlobClient blobClient = null;
IDictionary <string, string> dict = null;
Console.WriteLine("Enumerating blobs ...");
await foreach (BlobItem blobItem in contClient.GetBlobsAsync(BlobTraits.Metadata))
{
Console.WriteLine("-------------------------");
Console.WriteLine("\tName: " + blobItem.Name);
blobClient = contClient.GetBlobClient(blobItem.Name);
Console.WriteLine("\tUri: " + blobClient.Uri);
BlobItemProperties props = blobItem.Properties;
AccessTier?atier = props.AccessTier;
Console.WriteLine("\tAccess Tier: " + atier);
if (atier.Equals(SourceTier))
{
blobUrls[batchCount] = blobClient.Uri;
batchCount++;
}
;
dict = blobItem.Metadata;
if ((dict != null) && (dict.Count > 0))
{
foreach (KeyValuePair <string, string> item in dict)
{
Console.WriteLine("Key: {0}, Value: {1}", item.Key, item.Value);
}
}
else
{
Console.WriteLine("\tNo metadata for this blob item");
}
if (batchCount == BatchSize)
{
BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
totalProcessed += batchCount;
batchCount = 0;
Console.WriteLine("-------------------------");
Console.WriteLine($"No. of Blobs moved to {TargetTier} tier: {totalProcessed}");
if (taskList.Count >= ConcurrentTasks)
{
// Wait for existing tasks to finish before proceeeding.
// This is to avoid out of resources issue.
Task.WaitAll(taskList.ToArray());
taskList.Clear();
}
}
;
}
if (batchCount > 0)
{
BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
totalProcessed += batchCount;
}
;
if (taskList.Count > 0)
{
Task.WaitAll(taskList.ToArray()); // Wait for all Tasks to finish!
}
// Stop the timer
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}",
ts.Hours, ts.Minutes, ts.Seconds);
Console.WriteLine("------------------------------------");
Console.WriteLine($"Moved {totalProcessed} Blobs from {SourceTier} to {TargetTier} tier.");
Console.WriteLine($"Total Runtime: {elapsedTime}");
Console.WriteLine("------------------------------------");
}
catch (Exception ex)
{
Console.WriteLine("Encountered exception: " + ex);
}
}
