public void BatchErrors()
        {
            string connectionString = ConnectionString;
            string containerName    = Randomize("sample-container");

            #region Snippet:SampleSnippetsBatch_Troubleshooting
            // Get a connection string to our Azure Storage account.
            //@@ string connectionString = "<connection_string>";
            //@@ string containerName = "sample-container";

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(containerName);
            container.Create();

            // Create a blob named "valid"
            BlobClient valid = container.GetBlobClient("valid");
            valid.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Valid!")));

            // Get a reference to a blob named "invalid", but never create it
            BlobClient invalid = container.GetBlobClient("invalid");

            // Delete both blobs at the same time
            BlobBatchClient batch = service.GetBlobBatchClient();
            try
            {
                batch.DeleteBlobs(new Uri[] { valid.Uri, invalid.Uri });
            }
            catch (AggregateException)
            {
                // An aggregate exception is thrown for all the individual failures
                // Check ex.InnerExceptions for RequestFailedException instances
            }
            #endregion
        }
        public void BatchDelete()
        {
            string connectionString = ConnectionString;
            string containerName    = Randomize("sample-container");

            #region Snippet:SampleSnippetsBatch_DeleteBatch

            // Get a connection string to our Azure Storage account.
            //@@ string connectionString = "<connection_string>";
            //@@ string containerName = "sample-container";

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(containerName);
            container.Create();

            // Create three blobs named "foo", "bar", and "baz"
            BlobClient foo = container.GetBlobClient("foo");
            BlobClient bar = container.GetBlobClient("bar");
            BlobClient baz = container.GetBlobClient("baz");
            foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
            bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
            baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));

            // Delete all three blobs at once
            BlobBatchClient batch = service.GetBlobBatchClient();
            batch.DeleteBlobs(new Uri[] { foo.Uri, bar.Uri, baz.Uri });
            #endregion

            Assert.AreEqual(0, container.GetBlobs().ToList().Count);
            // Clean up after we're finished
            container.Delete();
        }
Exemple #3
0
        public async Task BatchSetAccessTierAsync()
        {
            // Get a connection string to our Azure Storage account.
            string connectionString = ConnectionString;

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
            await container.CreateAsync();

            try
            {
                // Create three blobs named "foo", "bar", and "baz"
                BlobClient foo = container.GetBlobClient("foo");
                BlobClient bar = container.GetBlobClient("bar");
                BlobClient baz = container.GetBlobClient("baz");
                await foo.UploadAsync(BinaryData.FromString("Foo!"));

                await bar.UploadAsync(BinaryData.FromString("Bar!"));

                await baz.UploadAsync(BinaryData.FromString("Baz!"));

                // Set the access tier for all three blobs at once
                BlobBatchClient batch = service.GetBlobBatchClient();
                await batch.SetBlobsAccessTierAsync(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool);
            }
            finally
            {
                // Clean up after the test when we're finished
                await container.DeleteAsync();
            }
        }
        public async Task BatchDeleteAsync()
        {
            // Get a connection string to our Azure Storage account.
            string connectionString = ConnectionString;

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
            await container.CreateAsync();

            try
            {
                // Create three blobs named "foo", "bar", and "baz"
                BlobClient foo = container.GetBlobClient("foo");
                BlobClient bar = container.GetBlobClient("bar");
                BlobClient baz = container.GetBlobClient("baz");
                await foo.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));

                await bar.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));

                await baz.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));

                // Delete all three blobs at once
                BlobBatchClient batch = service.GetBlobBatchClient();
                await batch.DeleteBlobsAsync(new Uri[] { foo.Uri, bar.Uri, baz.Uri });
            }
            finally
            {
                // Clean up after the test when we're finished
                await container.DeleteAsync();
            }
        }
        public void BatchSetAccessTierCool()
        {
            string connectionString = ConnectionString;
            string containerName    = Randomize("sample-container");

            #region Snippet:SampleSnippetsBatch_AccessTier
            // Get a connection string to our Azure Storage account.
            //@@ string connectionString = "<connection_string>";
            //@@ string containerName = "sample-container";

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(containerName);
            container.Create();
            // Create three blobs named "foo", "bar", and "baz"
            BlobClient foo = container.GetBlobClient("foo");
            BlobClient bar = container.GetBlobClient("bar");
            BlobClient baz = container.GetBlobClient("baz");
            foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
            bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
            baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));

            // Set the access tier for all three blobs at once
            BlobBatchClient batch = service.GetBlobBatchClient();
            batch.SetBlobsAccessTier(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool);
            #endregion

            foreach (BlobItem blob in container.GetBlobs())
            {
                Assert.AreEqual(AccessTier.Cool, blob.Properties.AccessTier);
            }
            // Clean up after the test when we're finished
        }
Exemple #6
0
        public void BatchSetAccessTier()
        {
            // Get a connection string to our Azure Storage account.
            string connectionString = ConnectionString;

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));

            container.Create();
            try
            {
                // Create three blobs named "foo", "bar", and "baz"
                BlobClient foo = container.GetBlobClient("foo");
                BlobClient bar = container.GetBlobClient("bar");
                BlobClient baz = container.GetBlobClient("baz");
                foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
                bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
                baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));

                // Set the access tier for all three blobs at once
                BlobBatchClient batch = service.GetBlobBatchClient();
                batch.SetBlobsAccessTier(new Uri[] { foo.Uri, bar.Uri, baz.Uri }, AccessTier.Cool);
            }
            finally
            {
                // Clean up after the test when we're finished
                container.Delete();
            }
        }
 /// <summary>
 /// Authenticate with <see cref="DefaultAzureCredential"/>.
 /// </summary>
 public void Authenticate()
 {
     #region Snippet:SampleSnippetsBlobBatch_Auth
     // Create a BlobServiceClient that will authenticate through Active Directory
     Uri accountUri           = new Uri("https://MYSTORAGEACCOUNT.blob.core.windows.net/");
     BlobServiceClient client = new BlobServiceClient(accountUri, new DefaultAzureCredential());
     BlobBatchClient   batch  = client.GetBlobBatchClient();
     #endregion
 }
Exemple #8
0
        /// <summary>
        /// Call the batch API with a bunch of URLs to change the tier of
        /// </summary>
        /// <param name="blobServiceClient">connection to blob service</param>
        /// <param name="uris">urls to move</param>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        private async Task ProcessBatch(BlobServiceClient blobServiceClient, IEnumerable <Uri> uris, CancellationToken cancellationToken)
        {
            _logger.LogInformation($"Sending Batch of {uris.Count()} items");

            using (var op = _telemetryClient.StartOperation <DependencyTelemetry>("ProcessBatch"))
            {
                op.Telemetry.Properties.Add("Run", _config.Run);
                op.Telemetry.Metrics.Add("BatchSize", uris.Count());

                if (!_config.WhatIf)
                {
                    BlobBatchClient batch = blobServiceClient.GetBlobBatchClient();
                    await batch.SetBlobsAccessTierAsync(uris, _targetAccessTier, cancellationToken : cancellationToken);
                }
            }
        }
Exemple #9
0
        public async Task Delete_Error()
        {
            // Arrange
            BlobServiceClient service = GetServiceClient_SharedKey();
            BlobServiceClient invalidServiceClient = InstrumentClient(new BlobServiceClient(
                                                                          GetServiceClient_SharedKey().Uri,
                                                                          GetOptions()));
            BlobBatchClient blobBatchClient = invalidServiceClient.GetBlobBatchClient();

            using BlobBatch batch = blobBatchClient.CreateBatch();
            batch.DeleteBlob(new Uri("https://account.blob.core.windows.net/container/blob"));

            // Act
            await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
                blobBatchClient.SubmitBatchAsync(batch),
                e => Assert.AreEqual(BlobErrorCode.NoAuthenticationInformation.ToString(), e.ErrorCode));
        }
        public async Task <bool> UpdateBlobStorageAccessSettingsArchive(string url)
        {
            try
            {
                BlobServiceClient cloudBlobServiceClient = new BlobServiceClient(_accessKey);
                // Set the access tier
                BlobBatchClient batch = cloudBlobServiceClient.GetBlobBatchClient();
                await batch.SetBlobsAccessTierAsync(new Uri[] { new Uri(url) }, AccessTier.Archive);

                return(true);
            }
            catch
            {
                //add logging
                throw;
            }
        }
Exemple #11
0
        public async Task FineGrainedBatchingAsync()
        {
            // Get a connection string to our Azure Storage account.
            string connectionString = ConnectionString;

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));
            await container.CreateAsync();

            try
            {
                // Create three blobs named "foo", "bar", and "baz"
                BlobClient foo = container.GetBlobClient("foo");
                BlobClient bar = container.GetBlobClient("bar");
                BlobClient baz = container.GetBlobClient("baz");
                await foo.UploadAsync(BinaryData.FromString("Foo!"));

                await bar.UploadAsync(BinaryData.FromString("Bar!"));

                await baz.UploadAsync(BinaryData.FromString("Baz!"));

                // Create a batch with three deletes
                BlobBatchClient batchClient = service.GetBlobBatchClient();
                BlobBatch       batch       = batchClient.CreateBatch();
                Response        fooResponse = batch.DeleteBlob(foo.Uri, DeleteSnapshotsOption.IncludeSnapshots);
                Response        barResponse = batch.DeleteBlob(bar.Uri);
                Response        bazResponse = batch.DeleteBlob(baz.Uri);

                // Submit the batch
                await batchClient.SubmitBatchAsync(batch);

                // Verify the results
                Assert.AreEqual(202, fooResponse.Status);
                Assert.AreEqual(202, barResponse.Status);
                Assert.AreEqual(202, bazResponse.Status);
            }
            finally
            {
                // Clean up after the test when we're finished
                await container.DeleteAsync();
            }
        }
        public void FineGrainedBatching()
        {
            string connectionString = ConnectionString;
            string containerName    = Randomize("sample-container");

            #region Snippet:SampleSnippetsBatch_FineGrainedBatching
            // Get a connection string to our Azure Storage account.
            //@@ string connectionString = "<connection_string>";
            //@@ string containerName = "sample-container";

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(containerName);
            container.Create();

            // Create three blobs named "foo", "bar", and "baz"
            BlobClient foo = container.GetBlobClient("foo");
            BlobClient bar = container.GetBlobClient("bar");
            BlobClient baz = container.GetBlobClient("baz");
            foo.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Foo!")));
            foo.CreateSnapshot();
            bar.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Bar!")));
            bar.CreateSnapshot();
            baz.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Baz!")));

            // Create a batch with three deletes
            BlobBatchClient batchClient = service.GetBlobBatchClient();
            BlobBatch       batch       = batchClient.CreateBatch();
            batch.DeleteBlob(foo.Uri, DeleteSnapshotsOption.IncludeSnapshots);
            batch.DeleteBlob(bar.Uri, DeleteSnapshotsOption.OnlySnapshots);
            batch.DeleteBlob(baz.Uri);

            // Submit the batch
            batchClient.SubmitBatch(batch);
            #endregion

            Pageable <BlobItem> blobs = container.GetBlobs(states: BlobStates.Snapshots);
            Assert.AreEqual(1, blobs.Count());
            Assert.AreEqual("bar", blobs.FirstOrDefault().Name);
            // Clean up after the test when we're finished
            container.Delete();
        }
        /*
         * Deletes all blobs in container
         */
        public static void DeleteAllBlobsUsingBatch(
            BlobServiceClient serviceClient,
            BlobContainerClient containerClient)
        {
            // Getting list of blob uris
            Pageable <BlobItem> allBlobs = containerClient.GetBlobs();

            Uri[] blobList = new Uri[allBlobs.Count()];
            int   count    = 0;

            foreach (BlobItem blob in allBlobs)
            {
                BlobClient blobClient = containerClient.GetBlobClient(blob.Name);
                blobList[count] = blobClient.Uri;
                count++;
            }

            // Creating batch client then setting access tier for all blobs
            BlobBatchClient batchClient = serviceClient.GetBlobBatchClient();

            batchClient.DeleteBlobs(blobList);
        }
        private async Task ProcessBatch(BlobServiceClient blobServiceClient, IEnumerable <Uri> uris, CancellationToken cancellationToken)
        {
            _logger.LogInformation($"Sending Batch of {uris.Count()} items");

            using (var op = _telemetryClient.StartOperation <DependencyTelemetry>("ProcessBatch"))
            {
                op.Telemetry.Properties.Add("Run", _config.Run);
                op.Telemetry.Metrics.Add("BatchSize", uris.Count());

                if (!_config.WhatIf)
                {
                    BlobBatchClient batch = blobServiceClient.GetBlobBatchClient();
                    if (_config.TargetAccessTier.Equals("Hot", StringComparison.InvariantCultureIgnoreCase))
                    {
                        await batch.SetBlobsAccessTierAsync(uris, AccessTier.Hot, cancellationToken : cancellationToken);
                    }
                    else
                    {
                        await batch.SetBlobsAccessTierAsync(uris, AccessTier.Cool, cancellationToken : cancellationToken);
                    }
                }
            }
        }
Exemple #15
0
        public void BatchErrors()
        {
            // Get a connection string to our Azure Storage account.
            string connectionString = ConnectionString;

            // Get a reference to a container named "sample-container" and then create it
            BlobServiceClient   service   = new BlobServiceClient(connectionString);
            BlobContainerClient container = service.GetBlobContainerClient(Randomize("sample-container"));

            container.Create();
            try
            {
                // Create a blob named "valid"
                BlobClient valid = container.GetBlobClient("valid");
                valid.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Valid!")));

                // Get a reference to a blob named "invalid", but never create it
                BlobClient invalid = container.GetBlobClient("invalid");

                // Delete both blobs at the same time
                BlobBatchClient batch = service.GetBlobBatchClient();
                batch.DeleteBlobs(new Uri[] { valid.Uri, invalid.Uri });
            }
            catch (AggregateException ex)
            {
                // An aggregate exception is thrown for all the indivudal failures
                Assert.AreEqual(1, ex.InnerExceptions.Count);
                StorageRequestFailedException failure = ex.InnerException as StorageRequestFailedException;
                Assert.IsTrue(BlobErrorCode.BlobNotFound == failure.ErrorCode);
            }
            finally
            {
                // Clean up after the test when we're finished
                container.Delete();
            }
        }
        static int SasKeyExpiryTime  = 1;                                                               // Default 1 hour

        static async Task Main(string[] args)
        {
            Console.WriteLine("Starting Blob operations");

            if (String.IsNullOrEmpty(AccountKey))
            {
                Console.WriteLine("The environment variable: AZURE_STORAGE_ACCOUNT_KEY, is not set!");
                Environment.Exit(-1);
            }
            ;

            if (SourceTier.Equals(TargetTier))
            {
                Console.WriteLine("Source and Target Access Tiers cannot be the same!");

                Environment.Exit(-1);
            }
            ;

            Stopwatch stopWatch = new Stopwatch();

            try
            {
                // Start the stop watch
                stopWatch.Start();

                BlobServiceClient   blobSvcClient = SharedAccessSignatureAuthAsync();
                BlobContainerClient contClient    = blobSvcClient.GetBlobContainerClient(ContainerName);

                Uri[] blobUrls       = new Uri[BatchSize];
                int   batchCount     = 0;
                int   totalProcessed = 0;
                List <Task <Azure.Response[]> > taskList = new List <Task <Azure.Response[]> >();

                BlobClient blobClient             = null;
                IDictionary <string, string> dict = null;
                Console.WriteLine("Enumerating blobs ...");
                await foreach (BlobItem blobItem in contClient.GetBlobsAsync(BlobTraits.Metadata))
                {
                    Console.WriteLine("-------------------------");
                    Console.WriteLine("\tName: " + blobItem.Name);
                    blobClient = contClient.GetBlobClient(blobItem.Name);
                    Console.WriteLine("\tUri: " + blobClient.Uri);

                    BlobItemProperties props = blobItem.Properties;

                    AccessTier?atier = props.AccessTier;
                    Console.WriteLine("\tAccess Tier: " + atier);
                    if (atier.Equals(SourceTier))
                    {
                        blobUrls[batchCount] = blobClient.Uri;
                        batchCount++;
                    }
                    ;

                    dict = blobItem.Metadata;
                    if ((dict != null) && (dict.Count > 0))
                    {
                        foreach (KeyValuePair <string, string> item in dict)
                        {
                            Console.WriteLine("Key: {0}, Value: {1}", item.Key, item.Value);
                        }
                    }
                    else
                    {
                        Console.WriteLine("\tNo metadata for this blob item");
                    }

                    if (batchCount == BatchSize)
                    {
                        BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
                        taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
                        totalProcessed += batchCount;
                        batchCount      = 0;

                        Console.WriteLine("-------------------------");
                        Console.WriteLine($"No. of Blobs moved to {TargetTier} tier: {totalProcessed}");

                        if (taskList.Count >= ConcurrentTasks)
                        {
                            // Wait for existing tasks to finish before proceeeding.
                            // This is to avoid out of resources issue.
                            Task.WaitAll(taskList.ToArray());
                            taskList.Clear();
                        }
                    }
                    ;
                }

                if (batchCount > 0)
                {
                    BlobBatchClient batch = blobSvcClient.GetBlobBatchClient();
                    taskList.Add(batch.SetBlobsAccessTierAsync(blobUrls, TargetTier));
                    totalProcessed += batchCount;
                }
                ;

                if (taskList.Count > 0)
                {
                    Task.WaitAll(taskList.ToArray()); // Wait for all Tasks to finish!
                }
                // Stop the timer
                stopWatch.Stop();
                TimeSpan ts          = stopWatch.Elapsed;
                string   elapsedTime = String.Format("{0:00}:{1:00}:{2:00}",
                                                     ts.Hours, ts.Minutes, ts.Seconds);
                Console.WriteLine("------------------------------------");
                Console.WriteLine($"Moved {totalProcessed} Blobs from {SourceTier} to {TargetTier} tier.");
                Console.WriteLine($"Total Runtime: {elapsedTime}");
                Console.WriteLine("------------------------------------");
            }
            catch (Exception ex)
            {
                Console.WriteLine("Encountered exception: " + ex);
            }
        }
        public static void Execute()
        {
            Console.WriteLine("\n Code demonstrating a Batch Delete scenario. \n");
            Console.WriteLine("\n [Warning this will delete all blobs in your account - Kindly type Yes to proceed] \n");

            // Take consent from the user (as it might get run by mistake from someone)
            string value = Console.ReadLine();

            if (value == "Yes")
            {
                Console.WriteLine("\n Proceeding with emptying all your blob containers!");
            }
            else
            {
                Console.WriteLine("\n Exiting  !");
                return;
            }

            // Set output path for debugging purposes
            Helper helper = new Helper();

            helper.SetConsoleOutPutPath(helper.RedirectOutputToFile, ".\\BatchDelete.txt");

            int segmentSize = 256;
            // Set up clients
            Random              random            = new Random();
            BlobServiceClient   blobServiceClient = new BlobServiceClient(helper.ConnectionString);
            BlobContainerClient container1        = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString());
            BlobContainerClient container2        = blobServiceClient.GetBlobContainerClient(helper.ContainerName + random.Next(0, 100).ToString());
            BlobBatchClient     batch             = blobServiceClient.GetBlobBatchClient();

            container1.CreateIfNotExists();
            container2.CreateIfNotExists();

            Console.WriteLine("\n Creating some sample blobs for deletion.");

            // Create 50 blobs in both the containers
            for (int i = 0; i < 50; i++)
            {
                Console.WriteLine("Creating blob id = {0}", i.ToString());

                BlobClient blob1 = container1.GetBlobClient("blob_" + i);
                blob1.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!")));

                BlobClient blob2 = container2.GetBlobClient("blob_" + i);
                blob2.Upload(new MemoryStream(Encoding.UTF8.GetBytes("Data!")));
            }

            // Call the listing operation and enumerate the result segment.
            var containerListResult =
                blobServiceClient.GetBlobContainers()
                .AsPages(null, segmentSize);

            foreach (var containerPage in containerListResult)
            {
                foreach (var containerItem in containerPage.Values)
                {
                    Console.WriteLine("Container name: {0}", containerItem.Name);
                    // Call the listing operation and return pages of the specified size.
                    var containerClient = blobServiceClient.GetBlobContainerClient(containerItem.Name);

                    var resultSegment = containerClient.GetBlobs()
                                        .AsPages(null, segmentSize);

                    // Enumerate the blobs returned for each page.
                    foreach (var blobPage in resultSegment)
                    {
                        List <Uri> Urilist = new List <Uri>();

                        foreach (var blobItem in blobPage.Values)
                        {
                            Console.WriteLine("Adding Blob to delete queue: {0}", blobItem.Name);
                            Urilist.Add(containerClient.GetBlobClient(blobItem.Name).Uri);
                        }

                        // Delete blobs at once
                        try
                        {
                            if (Urilist.Count > 0)
                            {
                                var response = batch.DeleteBlobs(Urilist);
                            }
                        }
                        catch (AggregateException ex)
                        {
                            Console.WriteLine(ex.Message.ToString());
                        }
                    }
                }
            }

            Console.WriteLine("All the blobs in the account are deleted successfully.");
        }