public static async Task <IEnumerable <CloudBlobContainer> > ListContainersAsync(string connectionString)
{
CloudStorageAccount.TryParse(connectionString, out CloudStorageAccount storageAccount);
if (storageAccount == null)
{
return(null);
}
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
if (storageAccount == null)
{
Console.WriteLine("Connection string did not work");
}
BlobContinuationToken continuationToken = null;
var containers = new List <CloudBlobContainer>();
do
{
ContainerResultSegment response = await cloudBlobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
containers.AddRange(response.Results);
} while (continuationToken != null);
return(containers);
}
public async Task <List <string> > ListFolders(string shareName)
{
List <string> toReturn = new List <string>();
try
{
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(this.ConnectionString);
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
BlobContinuationToken token = null;
do
{
ContainerResultSegment resultSegment = await cloudBlobClient.ListContainersSegmentedAsync(token);
token = resultSegment.ContinuationToken;
foreach (var item in resultSegment.Results)
{
toReturn.Add(item.Name);
}
} while (token != null);
}
catch (StorageException exStorage)
{
this.ErrorMessage = exStorage.ToString();
}
catch (Exception ex)
{
this.ErrorMessage = ex.ToString();
}
return(toReturn);
}
public async Task UpdateDwhSchemaAsync()
{
BlobContinuationToken token = null;
while (true)
{
ContainerResultSegment containersResult = await _blobClient.ListContainersSegmentedAsync(token);
if (containersResult?.Results != null)
{
foreach (var container in containersResult.Results)
{
await ProcessContainerAsync(container);
}
}
token = containersResult?.ContinuationToken;
if (token == null)
{
break;
}
}
if (_forcedUpdate)
{
_forcedUpdate = false;
}
_log.Info("Dwh structure update is finished");
}
internal static async Task <string> CreateBlobItem(string blobPath)
{
storageConnectionString = BaseConfiguration.Configuration["appsettings:storageConnectionString"];
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
string tempBlobPath = blobPath.ToLower();
if (File.Exists(tempBlobPath))
{
ContainerResultSegment containers = await blobClient.ListContainersSegmentedAsync(null);
CloudBlobContainer selectedContainer = containers.Results.Where(x => x.Name == "deeplibcontainer").FirstOrDefault();
if (selectedContainer != null)
{
CloudBlockBlob blob = selectedContainer.GetBlockBlobReference((DateTime.Now.Day.ToString("X") +
DateTime.Now.Month.ToString("X") + DateTime.Now.Year.ToString("X") +
DateTime.Now.Hour.ToString("X") + DateTime.Now.Minute.ToString("X") +
DateTime.Now.Second.ToString("X") + Path.GetExtension(blobPath)).ToLower());
await blob.UploadFromFileAsync(blobPath);
return(blob.Uri.ToString());
}
}
return(null);
}
public async Task <CloudContainersModel> ListContainers()
{
CloudContainersModel containersList = new CloudContainersModel();
try
{
var member = User.Identity.Name;
MemberLicense ml = this.iMemberLicenseRepository.MyLicense(member);
if (CloudStorageAccount.TryParse(ml?.AzureSaConnectionString, out CloudStorageAccount storageAccount))
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
BlobContinuationToken continuationToken = null;
do
{
ContainerResultSegment resultSegment = await blobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = resultSegment.ContinuationToken;
containersList.AddRange(resultSegment.Results);
}while (continuationToken != null);
}
}
catch
{
}
return(containersList);
}
private static async Task <CloudBlobContainer> GetContainerAsync(CloudBlobClient cloudBlobClient)
{
var containers = new List <CloudBlobContainer>();
ContainerResultSegment response = await cloudBlobClient.ListContainersSegmentedAsync(null);
containers.AddRange(response.Results);
return(containers.FirstOrDefault());
}
/// <summary>
/// Gets the children of the container (if the uri has segments)
/// or containers in the storage account (if the uri has no segments)
/// </summary>
public IEnumerable <Uri> GetChildren(Uri directoryUri)
{
string[] segments = directoryUri.Segments;
BlobContinuationToken blobContinuationToken = null;
// If at the root, return all containers
if (segments.Length <= 1)
{
do
{
ContainerResultSegment containerListing = _client.ListContainersSegmented(blobContinuationToken);
if (containerListing?.Results != null)
{
foreach (CloudBlobContainer containerItem in containerListing.Results)
{
yield return(containerItem.Uri);
}
}
blobContinuationToken = containerListing?.ContinuationToken;
}while (blobContinuationToken != null);
yield break;
}
// If not at the root folder, return all blobs within the container
string containerName = segments[1];
string relativeAddress = directoryUri.PathAndQuery.Substring(containerName.Length + 1);
do
{
BlobResultSegment listing = _client.ListBlobsSegmented(
containerName,
relativeAddress,
useFlatListing: false,
BlobListingDetails.None,
maxResults: null,
blobContinuationToken,
new BlobRequestOptions(),
new OperationContext());
if (listing == null || listing.Results.Count() == 0)
{
throw new ArgumentException(
"Call to ListBlobsSegmented returned no results. Uri is invalid or does not have children.",
nameof(directoryUri));
}
foreach (IListBlobItem listBlobItem in listing.Results)
{
yield return(listBlobItem.Uri);
}
blobContinuationToken = listing.ContinuationToken;
}while (blobContinuationToken != null);
}
private async Task <List <CloudBlobContainer> > GetCloudBlobContainersAsync(CancellationToken cancellationToken)
{
var result = new List <CloudBlobContainer>();
ContainerResultSegment firstPage = await NativeBlobClient.ListContainersSegmentedAsync(null);
result.AddRange(firstPage.Results);
//todo: list more containers
return(result);
}
public static async Task <List <CloudBlobContainer> > ListContainersAsync(CloudBlobClient blobClient)
{
BlobContinuationToken continuationToken = null;
List <CloudBlobContainer> containers = new List <CloudBlobContainer>();
do
{
ContainerResultSegment response = await blobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
containers.AddRange(response.Results);
} while (continuationToken != null);
return(containers);
}
protected override async Task Execute()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
string blobPath = "";
Console.Clear();
Console.WriteLine("-----Add Blob-----");
Console.WriteLine($"Blob path:");
do
{
blobPath = Console.ReadLine().ToLower();
if (System.IO.File.Exists(blobPath))
{
Console.WriteLine();
Console.WriteLine("Select target container:");
ContainerResultSegment containers = await blobClient.ListContainersSegmentedAsync(null);
int counter = 1;
foreach (var container in containers.Results)
{
Console.WriteLine($"{counter++}) {container.Name}");
}
Console.Write("Selected container: ");
string itemNumberInput = Console.ReadLine();
if (int.TryParse(itemNumberInput, out int itemNumber) && itemNumber > 0 && itemNumber < containers.Results.Count() + 1)
{
var selectedContainer = containers.Results.ElementAt(itemNumber - 1);
CloudBlockBlob blob = selectedContainer.GetBlockBlobReference(Path.GetFileName(blobPath));
await blob.UploadFromFileAsync((blobPath));
Console.WriteLine($"Blob uri: {blob.Uri}");
await TextCopy.Clipboard.SetTextAsync(blob.Uri.ToString());
}
}
else
{
Console.WriteLine("File does not exist. Try again");
Console.SetCursorPosition(0, Console.CursorTop - 2);
}
} while (!System.IO.File.Exists(blobPath));
}
private async Task <List <CloudBlobContainer> > GetCloudBlobContainersAsync(CancellationToken cancellationToken)
{
var result = new List <CloudBlobContainer>();
ContainerResultSegment page = null;
do
{
page = await NativeBlobClient.ListContainersSegmentedAsync(page?.ContinuationToken).ConfigureAwait(false);
result.AddRange(page.Results);
}while(page?.ContinuationToken != null);
return(result);
}
async private Task initBlogClient()
{
blobClient = storageAccount.CreateCloudBlobClient();
ContainerResultSegment resultSegment = null;
resultSegment = await blobClient.ListContainersSegmentedAsync(
"", ContainerListingDetails.Metadata, 5, continuationToken, null, null);
foreach (CloudBlobContainer container in resultSegment.Results)
{
collection.Add(container.Name);
}
listView.ItemsSource = collection;
}
private static async Task <IEnumerable <CloudBlobContainer> > ListContainersAsync(CloudBlobClient cloudBlobClient)
{
BlobContinuationToken continuationToken = null;
var containers = new List <CloudBlobContainer>();
do
{
ContainerResultSegment response = await cloudBlobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
containers.AddRange(response.Results);
} while (continuationToken != null);
return(containers);
}
public async Task <ObservableCollection <string> > ListContainersAsync()
{
BlobContinuationToken token = null;
ObservableCollection <string> _List = new ObservableCollection <string>();
do
{
ContainerResultSegment resultset = await _Client.ListContainersSegmentedAsync(token);
foreach (var item in resultset.Results)
{
_List.Add(item.Name);
}
} while (token != null);
return(_List);
}
private static async Task CleanupStorageAccount()
{
CloudBlobClient uatBlobClient = CloudStorageAccount.Parse(_options.UatConnectionString).CreateCloudBlobClient();
BlobContinuationToken containerCt = null;
do
{
ContainerResultSegment response = await uatBlobClient.ListContainersSegmentedAsync(containerCt);
foreach (CloudBlobContainer containerRef in response.Results.Where(t => !IsPlatform(t)))
{
await containerRef.DeleteIfExistsAsync();
}
containerCt = response.ContinuationToken;
} while (containerCt != null);
}
private void TestAccountAccess(string accountName, string accountKey, bool isDevStorage)
{
CloudStorageAccount account = null;
if (isDevStorage)
{
account = CloudStorageAccount.DevelopmentStorageAccount;
}
else
{
account = new CloudStorageAccount(new StorageCredentials(accountName, accountKey), EndpointDomain.Text, UseSSL);
}
CloudBlobClient blobClient = account.CreateCloudBlobClient();
// This will throw exception is the credential is incorrect
ContainerResultSegment containersSegmented = blobClient.ListContainersSegmented(null, ContainerListingDetails.None, 1, null);
}
protected override async Task Execute()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
Console.Clear();
Console.WriteLine("-----List of containers-----");
Console.WriteLine($"Containers list:");
ContainerResultSegment containers = await blobClient.ListContainersSegmentedAsync(null);
int counter = 1;
foreach (var container in containers.Results)
{
Console.WriteLine($"{counter++}) {container.Name}");
}
}
private static async Task ListContainersWithPrefixAsync(CloudBlobClient blobClient, string prefix)
{
Console.WriteLine("List all containers beginning with prefix {0}, plus container metadata:", prefix);
try
{
ContainerResultSegment resultSegment = null;
BlobContinuationToken continuationToken = null;
do
{
// List containers beginning with the specified prefix, returning segments of 5 results each.
// Passing null for the maxResults parameter returns the max number of results (up to 5000).
// Requesting the container's metadata with the listing operation populates the metadata,
// so it's not necessary to also call FetchAttributes() to read the metadata.
resultSegment = await blobClient.ListContainersSegmentedAsync(
prefix, ContainerListingDetails.Metadata, 5, continuationToken, null, null);
// Enumerate the containers returned.
foreach (var container in resultSegment.Results)
{
Console.WriteLine("\tContainer:" + container.Name);
// Write the container's metadata keys and values.
foreach (var metadataItem in container.Metadata)
{
Console.WriteLine("\t\tMetadata key: " + metadataItem.Key);
Console.WriteLine("\t\tMetadata value: " + metadataItem.Value);
}
}
// Get the continuation token. If not null, get the next segment.
continuationToken = resultSegment.ContinuationToken;
} while (continuationToken != null);
}
catch (StorageException e)
{
Console.WriteLine("HTTP error code {0} : {1}",
e.RequestInformation.HttpStatusCode,
e.RequestInformation.ErrorCode);
Console.WriteLine(e.Message);
}
}
public async Task <Guid[]> GetStoredContextIds()
{
var containers = new List <CloudBlobContainer>();
ContainerResultSegment results = null;
do
{
results = await _blobClient.ListContainersSegmentedAsync(
prefix : "ingest-",
currentToken : results?.ContinuationToken
).ConfigureAwait(false);
containers.AddRange(results.Results);
} while (results != null && results.ContinuationToken != null);
var ids = containers.Select(c => Guid.Parse(c.Name.Substring("ingest-".Length))).ToArray();
return(ids);
}
/// <inheritdoc/>
public async Task <List <string> > ListAllContainers()
{
var containerList = new List <string>();
CloudBlobClient cloudBlobClient = GetCloudBlobClient();
BlobContinuationToken token = null;
do
{
ContainerResultSegment containerResultSegment = await cloudBlobClient
.ListContainersSegmentedAsync(token);
token = containerResultSegment.ContinuationToken;
foreach (CloudBlobContainer cloudBlobContainer in containerResultSegment.Results)
{
containerList.Add(cloudBlobContainer.Name);
}
} while (token != null);
return(containerList);
}
public async Task <IEnumerable <string> > getListContainers()
{
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(Settings.storageConnectionString);
//create client
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
BlobContinuationToken continuationToken = null;
var containers = new List <CloudBlobContainer>();
do
{
ContainerResultSegment response = await cloudBlobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
containers.AddRange(response.Results);
} while (continuationToken != null);
var containersname = containers.Select(t => t.Name).Where(t => t != "$web");
return(containersname);
}
/// <summary>
/// List containers by container name prefix
/// </summary>
/// <param name="prefix">Container name prefix</param>
/// <returns>An enumerable collection of cloudblobcontainer</returns>
internal IEnumerable <Tuple <CloudBlobContainer, BlobContinuationToken> > ListContainersByPrefix(string prefix, Func <CloudBlobContainer, bool> containerFilter = null)
{
ContainerListingDetails details = ContainerListingDetails.Metadata;
BlobRequestOptions requestOptions = RequestOptions;
if (!string.IsNullOrEmpty(prefix) && !NameUtil.IsValidContainerPrefix(prefix))
{
throw new ArgumentException(String.Format(Resources.InvalidContainerName, prefix));
}
int listCount = InternalMaxCount;
int MaxListCount = 5000;
int requestCount = MaxListCount;
int realListCount = 0;
BlobContinuationToken continuationToken = ContinuationToken;
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
ContainerResultSegment containerResult = Channel.ListContainersSegmented(prefix, details, requestCount, continuationToken, requestOptions, OperationContext);
foreach (CloudBlobContainer container in containerResult.Results)
{
if (containerFilter == null || containerFilter(container))
{
yield return(new Tuple <CloudBlobContainer, BlobContinuationToken>(container, containerResult.ContinuationToken));
realListCount++;
}
}
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
continuationToken = containerResult.ContinuationToken;
}while (listCount > 0 && continuationToken != null);
}
public async Task ListContainers(string storageConnectionString)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
List <string> listedContainerNames = new List <string>();
BlobContinuationToken token = null;
do
{
ContainerResultSegment resultSegment = await blobClient.ListContainersSegmentedAsync(token);
token = resultSegment.ContinuationToken;
foreach (CloudBlobContainer container in resultSegment.Results)
{
listedContainerNames.Add(container.Name);
Console.WriteLine(" > " + container.Name);
}
}while (token != null);
}
public async Task <IEnumerable <string> > GetContainers()
{
List <string> containerNames = new List <string>();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConfig.ConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
BlobContinuationToken continuationToken = null;
ContainerResultSegment resultSegment = null;
do
{
resultSegment = await blobClient.ListContainersSegmentedAsync(continuationToken);
// Get the name of each blob.
containerNames.AddRange(resultSegment.Results.OfType <CloudBlobContainer>().Select(b => b.Name));
continuationToken = resultSegment.ContinuationToken;
} while (continuationToken != null);
return(containerNames);
}
public static async Task MigrateBlobs()
{
CloudBlobClient liveBlobClient = CloudStorageAccount.Parse(_options.LiveConnectionString).CreateCloudBlobClient();
CloudBlobClient uatBlobClient = CloudStorageAccount.Parse(_options.UatConnectionString).CreateCloudBlobClient();
BlobContinuationToken containerCt = null;
do
{
ContainerResultSegment response = await liveBlobClient.ListContainersSegmentedAsync(containerCt);
foreach (CloudBlobContainer sourceContainer in response.Results)
{
CloudBlobContainer destinationContainer = uatBlobClient.GetContainerReference(sourceContainer.Name);
Task labelTask = MigrateBlobs(destinationContainer, sourceContainer, "Label");
Task tileTask = MigrateBlobs(destinationContainer, sourceContainer, "Tile");
await Task.WhenAll(labelTask, tileTask);
}
containerCt = response.ContinuationToken;
} while (containerCt != null);
}
/// <summary>
/// Lista los contenedores
/// </summary>
public async Task <List <string> > ListContainersAsync(string prefix)
{
BlobContinuationToken continuationToken = null;
List <string> containers = new List <string>();
// Obtiene los contenedores
do
{
ContainerResultSegment resultSegment = await GetClient().ListContainersSegmentedAsync(prefix, ContainerListingDetails.Metadata, 100,
continuationToken, null, null);
// Enumera los contenedores
foreach (CloudBlobContainer container in resultSegment.Results)
{
containers.Add(container.Name);
}
// Obtiene el token de siguiente segmento
continuationToken = resultSegment.ContinuationToken;
}while (continuationToken != null);
// Devuelve la lista de contenedores
return(containers);
}
internal static async Task DeleteBlobItem(string blobUri)
{
storageConnectionString = BaseConfiguration.Configuration["appsettings:storageConnectionString"];
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
ContainerResultSegment containers = await blobClient.ListContainersSegmentedAsync(null);
CloudBlobContainer selectedContainer = containers.Results.Where(x => x.Name == "deeplibcontainer").FirstOrDefault();
BlobResultSegment blobs = await selectedContainer.ListBlobsSegmentedAsync(null);
CloudBlockBlob blob = (from CloudBlockBlob cloudBlob in blobs.Results
where cloudBlob.Uri.ToString() == blobUri
select cloudBlob).FirstOrDefault();
if (blob != null)
{
await blob.DeleteIfExistsAsync();
}
else
{
return;
}
}
protected override async Task Execute()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
string blobPath = "";
Console.Clear();
Console.WriteLine("-----Container Metadata-----");
Console.WriteLine($"Containers list:");
ContainerResultSegment containers = await blobClient.ListContainersSegmentedAsync(null);
int counter = 1;
foreach (var container in containers.Results)
{
Console.WriteLine($"{counter++}) {container.Name}");
}
Console.Write("Selected container: ");
string itemNumberInput = Console.ReadLine();
if (int.TryParse(itemNumberInput, out int itemNumber) && itemNumber > 0 && itemNumber < containers.Results.Count() + 1)
{
var selectedContainer = containers.Results.ElementAt(itemNumber - 1);
Console.WriteLine();
Console.WriteLine("Selected container metadata:");
foreach (var metadata in selectedContainer.Metadata)
{
Console.WriteLine($"{metadata.Key} - {metadata.Value}");
}
}
}
private void EnumerateContainers(string containerPrefix = "", bool testConnectivity = false)
{
BlobContinuationToken blobToken = new BlobContinuationToken();
ContainerResultSegment containerSegment = null;
string containerFilter = string.Empty;
if (!string.IsNullOrEmpty(_config.ContainerFilter))
{
containerPrefix = null;
containerFilter = _config.ContainerFilter;
}
try
{
Log.Info("account sas");
while (blobToken != null)
{
Log.Info($"containerPrefix:{containerPrefix} containerFilter:{containerFilter}");
containerSegment = _blobClient.ListContainersSegmentedAsync(containerPrefix, blobToken).Result;
if (testConnectivity)
{
return;
}
if (containerSegment.Results.Any())
{
IEnumerable <CloudBlobContainer> containers = containerSegment.Results.Where(x => Regex.IsMatch(x.Name, containerFilter, RegexOptions.IgnoreCase));
Log.Info("container list results:", containers.Select(x => x.Name));
if (containers.Count() < 1)
{
Log.Warning($"no containers detected. use 'containerFilter' argument to specify");
}
if (containers.Count() > 1)
{
Log.Warning($"multiple containers detected. use 'containerFilter' argument to enumerate only one");
}
foreach (CloudBlobContainer container in containers)
{
AddContainerToList(container);
}
blobToken = containerSegment.ContinuationToken;
}
else if (!string.IsNullOrEmpty(containerPrefix))
{
Log.Warning("retrying without containerPrefix");
containerPrefix = null;
}
else
{
Log.Warning("no results");
break;
}
}
return;
}
catch (Exception e)
{
Log.Debug($"{e}");
Log.Warning($"unable to connect to containerPrefix: {containerPrefix} containerFilter: {containerFilter} error: {e.HResult}");
}
if (_config.SasEndpointInfo.AbsolutePath.Length > 1)
{
Log.Info("absolute path sas");
CloudBlobContainer container = new CloudBlobContainer(new Uri(_account.BlobEndpoint + _config.SasEndpointInfo.AbsolutePath + "?" + _account.Credentials.SASToken));
// force connection / error
if (container.ListBlobsSegmented(null, true, new BlobListingDetails(), 1, null, null, null).Results.Count() == 1)
{
if (testConnectivity)
{
return;
}
Log.Info($"connected with absolute path to container:{container.Name}", ConsoleColor.Green);
AddContainerToList(container);
}
else
{
string errMessage = $"there are no blobs in container:{container.Name}";
Log.Error(errMessage);
throw new Exception(errMessage);
}
}
else
{
string errMessage = "unable to enumerate containers with or without absolute path";
Log.Error(errMessage);
throw new Exception(errMessage);
}
}
public FifthweekContainerResultSegment(ContainerResultSegment segment)
{
this.segment = segment;
}
