public async Task <List <ContainerInfoViewModel> > ContainerList()
{
try
{
List <ContainerInfoViewModel> containers = new List <ContainerInfoViewModel>();
var resultSegment = blobServiceClient.GetBlobContainersAsync().AsPages();
await foreach (Azure.Page <BlobContainerItem> containerPage in resultSegment)
{
foreach (BlobContainerItem containerItem in containerPage.Values)
{
ContainerInfoViewModel newContainer = new ContainerInfoViewModel()
{
Name = containerItem.Name,
Properties = containerItem.Properties
};
containers.Add(newContainer);
}
}
return(containers);
}
catch (RequestFailedException e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
throw;
}
}
public async Task CollectGarbage_Explicit()
{
// Get all of our configs
var configs = new List <TenantConfiguration>();
try { configs.Add(TestConfigurations.DefaultTargetTenant); } catch (InconclusiveException) { }
try { configs.Add(TestConfigurations.DefaultSecondaryTargetTenant); } catch (InconclusiveException) { }
try { configs.Add(TestConfigurations.DefaultTargetPremiumBlobTenant); } catch (InconclusiveException) { }
try { configs.Add(TestConfigurations.DefaultTargetPreviewBlobTenant); } catch (InconclusiveException) { }
try { configs.Add(TestConfigurations.DefaultTargetOAuthTenant); } catch (InconclusiveException) { }
foreach (TenantConfiguration config in configs)
{
// Blobs
var blobs = new BlobServiceClient(config.ConnectionString);
await foreach (BlobContainerItem container in blobs.GetBlobContainersAsync())
{
try
{
await blobs.DeleteBlobContainerAsync(container.Name);
}
catch (StorageRequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.LeaseIdMissing)
{
// Break any lingering leases
await blobs.GetBlobContainerClient(container.Name).GetLeaseClient().BreakAsync();
}
catch (StorageRequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.ContainerBeingDeleted)
{
// Ignore anything already being deleted
}
}
// Queues
var queues = new QueueServiceClient(config.ConnectionString);
await foreach (QueueItem queue in queues.GetQueuesAsync())
{
try
{
await queues.DeleteQueueAsync(queue.Name);
}
catch (StorageRequestFailedException ex) when(ex.ErrorCode == QueueErrorCode.QueueBeingDeleted)
{
// Ignore anything already being deleted
}
}
// Files
var files = new FileServiceClient(config.ConnectionString);
await foreach (ShareItem share in files.GetSharesAsync())
{
try
{
await files.DeleteShareAsync(share.Name);
}
catch (StorageRequestFailedException ex) when(ex.ErrorCode == FileErrorCode.ShareBeingDeleted)
{
// Ignore anything already being deleted
}
}
}
}
public async IAsyncEnumerable <string> GetAsync()
{
await foreach (BlobContainerItem container in _client.GetBlobContainersAsync())
{
yield return(container.Name);
}
}
private async Task <IReadOnlyCollection <BlobContainerClient> > ListContainersAsync(CancellationToken cancellationToken)
{
var r = new List <BlobContainerClient>();
//check that the special "$logs" container exists
BlobContainerClient logsContainerClient = _client.GetBlobContainerClient("$logs");
Task <Response <BlobContainerProperties> > logsProps = logsContainerClient.GetPropertiesAsync();
//in the meanwhile, enumerate
await foreach (BlobContainerItem container in _client.GetBlobContainersAsync(BlobContainerTraits.Metadata).ConfigureAwait(false))
{
(BlobContainerClient client, _) = await GetPartsAsync(container.Name, false).ConfigureAwait(false);
if (client != null)
{
r.Add(client);
}
}
try
{
await logsProps.ConfigureAwait(false);
r.Add(logsContainerClient);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "ContainerNotFound")
{
}
return(r);
}
public async Task UndeleteBlobContainerAsync()
{
// Arrange
BlobServiceClient service = GetServiceClient_SoftDelete();
string containerName = GetNewContainerName();
BlobContainerClient container = InstrumentClient(service.GetBlobContainerClient(containerName));
await container.CreateAsync();
await container.DeleteAsync();
IList <BlobContainerItem> containers = await service.GetBlobContainersAsync(states : BlobContainerStates.Deleted).ToListAsync();
BlobContainerItem containerItem = containers.Where(c => c.Name == containerName).FirstOrDefault();
// It takes some time for the Container to be deleted.
await Delay(30000);
// Act
Response <BlobContainerClient> response = await service.UndeleteBlobContainerAsync(
containerItem.Name,
containerItem.VersionId,
GetNewContainerName());
// Assert
await response.Value.GetPropertiesAsync();
// Cleanup
await container.DeleteAsync();
}
private static async Task EnumerateContainersAsync(BlobServiceClient client)
{
await foreach (BlobContainerItem container in client.GetBlobContainersAsync())
{
await Console.Out.WriteLineAsync($"Container:\t{container.Name}");
}
}
public async Task ListContainersSegmentAsync()
{
// Arrange
BlobServiceClient service = GetServiceClient_SharedKey();
// Ensure at least one container
await using DisposingContainer test = await GetTestContainerAsync(service : service);
// Act
IList <BlobContainerItem> containers = await service.GetBlobContainersAsync().ToListAsync();
// Assert
Assert.IsTrue(containers.Count() >= 1);
var accountName = new BlobUriBuilder(service.Uri).AccountName;
TestHelper.AssertCacheableProperty(accountName, () => service.AccountName);
Assert.IsNotNull(containers[0].Name);
Assert.IsNotNull(containers[0].Properties);
Assert.IsNotNull(containers[0].Properties.ETag);
Assert.IsNotNull(containers[0].Properties.HasImmutabilityPolicy);
Assert.IsNotNull(containers[0].Properties.HasLegalHold);
Assert.IsNotNull(containers[0].Properties.LastModified);
Assert.IsNotNull(containers[0].Properties.LeaseState);
Assert.IsNotNull(containers[0].Properties.LeaseStatus);
if (_serviceVersion >= BlobClientOptions.ServiceVersion.V2019_07_07)
{
Assert.IsNotNull(containers[0].Properties.DefaultEncryptionScope);
Assert.IsNotNull(containers[0].Properties.PreventEncryptionScopeOverride);
}
}
public async Task <List <FileResponseViewModel> > BlobListByContainersAsync(List <string> containers)
{
if (containers.Count == 0)
{
var resultSegment = blobServiceClient.GetBlobContainersAsync().AsPages();
await foreach (Azure.Page <BlobContainerItem> containerPage in resultSegment)
{
foreach (BlobContainerItem containerItem in containerPage.Values)
{
containers.Add(containerItem.Name);
}
}
}
// List all the blobs
List <FileResponseViewModel> blobs = new List <FileResponseViewModel>();
foreach (string containerName in containers)
{
BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName);
await foreach (BlobItem blob in container.GetBlobsAsync())
{
blobs.Add(new FileResponseViewModel(blob, containerName));
}
}
return(blobs);
}
private async Task EnumerateContainers()
{
Console.WriteLine($"Enumerating containers in storage account '{_serviceClient.AccountName}'");
await foreach (var container in _serviceClient.GetBlobContainersAsync())
{
Console.WriteLine($"\t{container.Name}");
}
}
//Create a new static method named EnumerateContainersAsync with a single BlobServiceClient parameter
private static async Task EnumerateContainersAsync(BlobServiceClient client)
{
//loop over all containers and print out the name
await foreach (BlobContainerItem container in client.GetBlobContainersAsync())
{
await Console.Out.WriteLineAsync($"Contains:\t{container.Name}");
}
}
public async Task <List <BlobContainerItem> > ListContainers(BlobServiceClient blobServiceClient, string prefix)
{
try
{
// Call the listing operation and enumerate the result segment.
var resultSegment =
blobServiceClient.GetBlobContainersAsync(BlobContainerTraits.Metadata, prefix, default)
.AsPages(default);
private async Task DeleteAllContainersAsync(string prefix)
{
var pagenable = blobClient.GetBlobContainersAsync(prefix: prefix);
await foreach (var blobItem in pagenable)
{
await blobClient.DeleteBlobContainerAsync(blobItem.Name);
}
}
public async Task <List <string> > GetContainersListAsync()
{
var result = new List <string>();
await foreach (var containerItem in client.GetBlobContainersAsync())
{
result.Add(containerItem.Name);
}
return(result);
}
// Storage
private static async Task <String> EnumerateContainersAsync(BlobServiceClient client)
{
var containers = "";
await foreach (BlobContainerItem container in client.GetBlobContainersAsync())
{
containers += container.Name;
}
return(containers);
}
/// <summary>
/// List blob containers in a storage account.
/// </summary>
/// <returns>List of blob containers.</returns>
public static async Task <IEnumerable <string> > ListContainers()
{
List <string> containers = new List <string>();
await foreach (var container in s_blobServiceClient.GetBlobContainersAsync())
{
containers.Add(container.Name);
}
return(containers);
}
//-------------------------------------------------
// List containers
//-------------------------------------------------
// <Snippet_ListContainers>
async static Task ListContainers(BlobServiceClient blobServiceClient,
string prefix,
int?segmentSize)
{
try
{
// Call the listing operation and enumerate the result segment.
var resultSegment =
blobServiceClient.GetBlobContainersAsync(BlobContainerTraits.Metadata, prefix, default)
.AsPages(default, segmentSize);
public async Task ListContainersSegmentAsync_Error()
{
// Arrange
BlobServiceClient service = GetServiceClient_SharedKey();
// Act
await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
service.GetBlobContainersAsync().AsPages(continuationToken : "garbage").FirstAsync(),
e => Assert.AreEqual("OutOfRangeInput", e.ErrorCode));
}
public async Task OnExecute(IConsole console)
{
var sw = new Stopwatch();
sw.Start();
if (string.IsNullOrEmpty(ConnectionString))
{
ConnectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONNECTION_STRING");
if (string.IsNullOrEmpty(ConnectionString))
{
throw new ArgumentException(nameof(ConnectionString));
}
}
if (!All && string.IsNullOrEmpty(ContainerName))
{
throw new ArgumentException("need the container name");
}
var csa = CloudStorageAccount.Parse(ConnectionString);
var account = new CloudStorageAccount(csa.Credentials, csa.TableStorageUri);
var bc = new BlobServiceClient(ConnectionString);
var token = string.Empty;
int count = 0;
do
{
var results = bc.GetBlobContainersAsync(BlobContainerTraits.Metadata)
.AsPages(token);
await foreach (var cp in results)
{
foreach (var item in cp.Values)
{
if (All)
{
await bc.DeleteBlobContainerAsync(item.Name);
count++;
}
else
{
if (item.Name.Equals(ContainerName))
{
await bc.DeleteBlobContainerAsync(item.Name);
}
count++;
}
}
}
} while (token != string.Empty);
sw.Stop();
console.WriteLine($"deleted {count} containers in {sw.Elapsed.TotalSeconds} seconds.");
}
public async Task <string[]> ListContainers()
{
var names = new List <string>();
var contains = blobServiceClient.GetBlobContainersAsync();
await foreach (var v in contains)
{
names.Add(v.Name);
}
return(names.ToArray());
}
static async Task EnumerateBlobContainerAsync(BlobServiceClient client)
{
int i = 1;
await foreach (BlobContainerItem container in client.GetBlobContainersAsync())
{
await Console.Out.WriteLineAsync($"Container {i++}:\t{container.Name}");
await EnumerateBlobsAsync(client, container.Name);
}
}
private static async Task Clean(BlobServiceClient blobClient, QueueServiceClient queueClient)
{
await foreach (var testContainer in blobClient.GetBlobContainersAsync(prefix: TestArtifactPrefix))
{
await blobClient.GetBlobContainerClient(testContainer.Name).DeleteAsync();
}
await foreach (var queue in queueClient.GetQueuesAsync(prefix: TestArtifactPrefix))
{
await queueClient.GetQueueClient(queue.Name).DeleteAsync();
}
}
private async Task <BlobContainerClient> SearchBlobContainer()
{
await foreach (BlobContainerItem container in _blobServiceClient.GetBlobContainersAsync())
{
if (container.Name.Equals(_containerName.Trim()))
{
BlobContainerClient blobContainer = _blobServiceClient.GetBlobContainerClient(_containerName);
Console.WriteLine("Blob Container {0} was found.", _containerName);
return(blobContainer);
}
}
return(null);
}
public async Task DisposeAsync()
{
if (Host != null)
{
await Host.StopAsync();
VerifyLockState("WebJobs.Internal.Blobs.Listener", LeaseState.Available, LeaseStatus.Unlocked).Wait();
await foreach (var testContainer in BlobServiceClient.GetBlobContainersAsync(prefix: TestArtifactPrefix))
{
await BlobServiceClient.GetBlobContainerClient(testContainer.Name).DeleteAsync();
}
}
}
public async Task <ActionResult <JArray> > GetStorageDfsFilesAsync(Guid engineId, string dataSourceName)
{
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
var dataSourceResponse = await this.dataFactoriesController.GetDataSourceAsync(engineId, dataSourceName);
if (dataSourceResponse.Value == null)
{
throw new Exception("DataSource does not exists");
}
var dataSource = dataSourceResponse.Value;
if (dataSource.DataSourceType != YDataSourceType.AzureBlobFS && dataSource.DataSourceType != YDataSourceType.AzureBlobStorage)
{
throw new Exception("DataSource is not a Azure Blob or Azure Data Lake Gen2 Data Source.");
}
// Get typed instance to get the correct call to GetSensitiveString()
var typeDataSource = YDataSourceFactory.GetTypedDatSource(dataSource) as YDataSourceAzureBlob;
var accountKey = await this.keyVaultsController.GetKeyVaultSecret(engineId, dataSource.Name);
if (accountKey == null || accountKey.Value == null)
{
throw new Exception("DataSource Account key is not present in th keyvault");
}
StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(typeDataSource.StorageAccountName, accountKey.Value);
var root = new JArray();
if (typeDataSource.DataSourceType == YDataSourceType.AzureBlobStorage)
{
// be careful, to get account detail, we are targeting ".bob." and not ".dfs"
string dfsUri = "https://" + typeDataSource.StorageAccountName + ".blob.core.windows.net";
var blobServiceClient = new BlobServiceClient(new Uri(dfsUri), sharedKeyCredential);
AsyncPageable <BlobContainerItem> allContainers = blobServiceClient.GetBlobContainersAsync();
await foreach (var containerItem in allContainers)
{
var blobContainerClient = blobServiceClient.GetBlobContainerClient(containerItem.Name);
ListBlobsHierarchicalListing(blobContainerClient, default, default, ref root);
public async Task <IEnumerable <Container> > ListContainerAsync()
{
var lstContainer = new List <Container>();
await foreach (BlobContainerItem container in _blobServiceClient.GetBlobContainersAsync())
{
var nose = new Container
{
Name = container.Name
};
lstContainer.Add(nose);
}
return(lstContainer);
}
private static async Task ListContainerFilesAsync()
{
var blobServiceClient = new BlobServiceClient(_connectionString);
await foreach (var blobContainerItem in blobServiceClient.GetBlobContainersAsync())
{
Console.WriteLine($"\t{blobContainerItem.Name}");
var blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerItem.Name);
await foreach (BlobItem blobItem in blobContainerClient.GetBlobsAsync())
{
Console.WriteLine($"\t - {blobItem.Name}");
}
}
}
private async Task <TestExceptionPolicy> PerformSecondaryStorageTest(int numberOfReadFailuresToSimulate, bool retryOn404 = false)
{
BlobServiceClient service = GetServiceClient_SecondaryAccount_ReadEnabledOnRetry(
numberOfReadFailuresToSimulate,
out TestExceptionPolicy testExceptionPolicy,
retryOn404);
await using DisposingContainer test = await GetTestContainerAsync(service : service);
IList <BlobContainerItem> containers = await EnsurePropagatedAsync(
async() => await service.GetBlobContainersAsync().ToListAsync(),
containers => containers.Count > 0);
Assert.IsTrue(containers.Count >= 1);
return(testExceptionPolicy);
}
public static async Task <List <CloudBlobContainerWrapper> > ListContainersAsync(string account, string key)
{
BlobServiceClient blobServiceClient = new BlobServiceClient(Client.GetConnectionString(account, key));
List <CloudBlobContainerWrapper> results = new List <CloudBlobContainerWrapper>();
await foreach (var container in blobServiceClient.GetBlobContainersAsync())
{
results.Add(new CloudBlobContainerWrapper
{
Name = container.Name
});
}
return(results);
}
private async Task EnumerateContainersAnonymously()
{
var anonServiceClient = new BlobServiceClient(this.serviceClient.Uri);
Console.WriteLine($"Anonymously enumerating containers in storage account '{this.serviceClient.AccountName}'");
try
{
await foreach (var container in anonServiceClient.GetBlobContainersAsync())
{
Console.WriteLine($"\t{container.Name}");
}
}
catch (Exception ex)
{
Console.WriteLine($"\tFailed with {ex.GetType().ToString()}");
}
}
public async Task ListContainersSegmentAsync()
{
// Arrange
BlobServiceClient service = GetServiceClient_SharedKey();
// Ensure at least one container
await using DisposingContainer test = await GetTestContainerAsync(service : service);
// Act
IList <BlobContainerItem> containers = await service.GetBlobContainersAsync().ToListAsync();
// Assert
Assert.IsTrue(containers.Count() >= 1);
var accountName = new BlobUriBuilder(service.Uri).AccountName;
TestHelper.AssertCacheableProperty(accountName, () => service.AccountName);
}
