public async Task <TerraModuleCollection> GetAllModulesAsync()
{
var modules = new List <TerraModule>();
var segments = await blobClient.ListContainersSegmentedAsync(default(BlobContinuationToken));
foreach (var bc in segments.Results)
{
var blobSegs = await bc.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All,
100, default(BlobContinuationToken), null, null);
// Loop over items within the container and output the length and URI.
foreach (var item in blobSegs.Results)
{
if (item.GetType() == typeof(CloudBlockBlob))
{
var blob = (CloudBlockBlob)item;
// azure/api-app/0.0.44176.zip
var parts = blob.Name.Split("/".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
var @namespace = blob.Container.Name;
var name = "Unknown";
var provider = "Unknown";
var version = "Unknown";
if (parts.Length == 3)
{
name = parts[1];
provider = parts[0];
version = $"{parts[2]}".ToLowerInvariant().Replace(".zip", string.Empty);
}
modules.Add(new TerraModule
{
Id = $"{@namespace}/{name}/{provider}/{version}",
Owner = string.Empty,
Namespace = @namespace,
Name = name,
Version = version,
Provider = provider,
Description = $"{@namespace}/{name}/{provider}/{version}",
Downloads = new Random().Next(10, 30),
Published_at = DateTime.Now.Subtract(new TimeSpan(22, 0, 0, 0)),
Source = $"https://github.com/{ @namespace }/{ name }/{ provider }/{ version }",
Verified = true
});
}
}
}
return(await Task.FromResult(new TerraModuleCollection
{
Meta = new TerraMetaData
{
Current_offset = 0,
Limit = modules.Count,
Next_offset = -1,
Next_url = string.Empty
},
Modules = modules
}));
}
internal async Task AddContainersToRoot(DirectoryBlock directory)
{
var items = await client.ListContainersSegmentedAsync(null, new ContainerListingDetails(),
maxListResults,
new BlobContinuationToken(),
new BlobRequestOptions(),
new OperationContext());
var list_continue = false;
do
{
if (items != null)
{
foreach (var item in items.Results)
{
AddToParentDirectoryObject(directory, item);
}
}
if (items.ContinuationToken != null)
{
items = await client.ListContainersSegmentedAsync(null, new ContainerListingDetails(), maxListResults, items.ContinuationToken, new BlobRequestOptions(), new OperationContext());
list_continue = true;
}
else
{
list_continue = false;
}
} while (list_continue);
}
internal static async Task <string> CreateBlobItem(string blobPath)
{
storageConnectionString = BaseConfiguration.Configuration["appsettings:storageConnectionString"];
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
string tempBlobPath = blobPath.ToLower();
if (File.Exists(tempBlobPath))
{
ContainerResultSegment containers = await blobClient.ListContainersSegmentedAsync(null);
CloudBlobContainer selectedContainer = containers.Results.Where(x => x.Name == "deeplibcontainer").FirstOrDefault();
if (selectedContainer != null)
{
CloudBlockBlob blob = selectedContainer.GetBlockBlobReference((DateTime.Now.Day.ToString("X") +
DateTime.Now.Month.ToString("X") + DateTime.Now.Year.ToString("X") +
DateTime.Now.Hour.ToString("X") + DateTime.Now.Minute.ToString("X") +
DateTime.Now.Second.ToString("X") + Path.GetExtension(blobPath)).ToLower());
await blob.UploadFromFileAsync(blobPath);
return(blob.Uri.ToString());
}
}
return(null);
}
public async Task <IEnumerable <IStorageBucket> > ListBucketsAsync()
{
var segmentedResult = await _blobClient.ListContainersSegmentedAsync(
string.Empty, ContainerListingDetails.Metadata, null, null, null, null);
return(segmentedResult.Results.Select(c => new AzureBlobStorageContainer(this, c)));
}
public async Task <IReadOnlyList <IUnixFileSystemEntry> > GetEntriesAsync(IUnixDirectoryEntry directoryEntry, CancellationToken cancellationToken)
{
var ret = new List <IUnixFileSystemEntry>();
//handle container listing
if (directoryEntry.IsRoot == true)
{
var containers = _blobClient.ListContainersSegmentedAsync(null).Result;
ret.AddRange(containers.Results.Select(c => new AzureStorageDirectoryEntry(c)));
}
//handle a url inside a container
else
{
var split = directoryEntry.Name.Split('/').Where(s => string.IsNullOrEmpty(s) == false).ToArray();
if (split != null && split.Length > 0)
{
var container = _blobClient.GetContainerReference(split.FirstOrDefault());
var subdir = split.Length > 1 ? string.Join("/", split.Skip(1).Take(split.Length - 1)) : "";
var directoryReference = container.GetDirectoryReference(subdir);
//get blobs in dir
var blobs = await directoryReference.ListBlobsSegmentedAsync(false, BlobListingDetails.Copy, 10000, null, null, null);
ret.AddRange(blobs.Results.Where(b => b.GetType() == typeof(CloudBlobDirectory)).Select(b => new AzureStorageDirectoryEntry(b as CloudBlobDirectory)));
ret.AddRange(blobs.Results.Where(b => b.GetType() != typeof(CloudBlobDirectory)).Select(b => new AzureStorageFileEntry(b as CloudBlockBlob)));
}
}
return(ret);
}
private async Task <JArray> GetContainersAsync(string storageAccountConnectionString)
{
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(storageAccountConnectionString);
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
BlobContinuationToken continuationToken = null;
var array = new JArray();
do
{
var response = await blobClient.ListContainersSegmentedAsync(continuationToken);
foreach (var container in response.Results)
{
JObject containerObject = new JObject
{
{ "Name", container.Name }
};
array.Add(containerObject);
}
}while (continuationToken != null);
return(array);
}
private static async Task <TResult> FindAllContainersAsync <TResult>(this CloudBlobClient sourceClient, Func <bool> stopCalled, Func <CloudBlobContainer[], TResult> onSuccess, Func <string, TResult> onFailure)
{
var context = new OperationContext();
BlobContinuationToken token = null;
var containers = new List <CloudBlobContainer>();
while (true)
{
if (stopCalled())
{
return(onFailure($"listing containers stopped on {sourceClient.Credentials.AccountName}"));
}
try
{
var segment = await sourceClient.ListContainersSegmentedAsync(null, ContainerListingDetails.All,
null, token, BlobCopyOptions.requestOptions, context);
var results = segment.Results.ToArray();
containers.AddRange(results);
token = segment.ContinuationToken;
if (null == token)
{
return(onSuccess(containers.ToArray()));
}
}
catch (Exception e)
{
return(onFailure($"Exception listing all containers, Detail: {e.Message}"));
}
}
}
static async System.Threading.Tasks.Task Main(string[] args)
{
string storageConnectionString = "DefaultEndpointsProtocol=https;AccountName=strrdm;AccountKey=jykbgp7GhLpHiqZIMtstB+tTLx+EtfELlrOHhPbNV4PUHDySga97Rv/jK68v9mSrFx6INL/jVFJpzv51uyWvqQ==;EndpointSuffix=core.windows.net";
CloudStorageAccount storageAccount;
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{// Create the CloudBlobClient that represents the Blob storage endpoint for the storage account.
CloudBlobClient cbc = storageAccount.CreateCloudBlobClient();
var conroot = cbc.GetRootContainerReference();
Console.WriteLine(conroot.Name);
BlobContinuationToken token = null;
var listcon = new List <CloudBlobContainer>();
do
{
var respon = await cbc.ListContainersSegmentedAsync(token);
listcon.AddRange(respon.Results);
token = respon.ContinuationToken;
} while (token != null);
foreach (var c in listcon)
{
Console.WriteLine(c.Name);
}
Console.WriteLine(listcon.Count);
}
}
public async Task <List <string> > ListFolders(string shareName)
{
List <string> toReturn = new List <string>();
try
{
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(this.ConnectionString);
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
BlobContinuationToken token = null;
do
{
ContainerResultSegment resultSegment = await cloudBlobClient.ListContainersSegmentedAsync(token);
token = resultSegment.ContinuationToken;
foreach (var item in resultSegment.Results)
{
toReturn.Add(item.Name);
}
} while (token != null);
}
catch (StorageException exStorage)
{
this.ErrorMessage = exStorage.ToString();
}
catch (Exception ex)
{
this.ErrorMessage = ex.ToString();
}
return(toReturn);
}
public async Task <List <AzureStorageContainer> > GetContainers()
{
var list = new List <AzureStorageContainer>();
var prefix = string.Empty;
var continuationToken = new BlobContinuationToken();
var requestOptions = new BlobRequestOptions();
var cancelToken = new CancellationToken();
var opContext = new OperationContext();
var operationOutcome = await blobClient.ListContainersSegmentedAsync(prefix, ContainerListingDetails.All, 1000, continuationToken, requestOptions, opContext, cancelToken);
foreach (var item in operationOutcome.Results)
{
var container = new AzureStorageContainer();
container.Name = item.Name;
var accessLevel = await GetAccessLevel(container);
container.AccessLevel = new ContainerAccessLevel()
{
Code = accessLevel
};
list.Add(container);
}
return(list);
}
public static async Task <IEnumerable <CloudBlobContainer> > ListContainersAsync(string connectionString)
{
CloudStorageAccount.TryParse(connectionString, out CloudStorageAccount storageAccount);
if (storageAccount == null)
{
return(null);
}
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
if (storageAccount == null)
{
Console.WriteLine("Connection string did not work");
}
BlobContinuationToken continuationToken = null;
var containers = new List <CloudBlobContainer>();
do
{
ContainerResultSegment response = await cloudBlobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
containers.AddRange(response.Results);
} while (continuationToken != null);
return(containers);
}
public async Task UpdateDwhSchemaAsync()
{
BlobContinuationToken token = null;
while (true)
{
ContainerResultSegment containersResult = await _blobClient.ListContainersSegmentedAsync(token);
if (containersResult?.Results != null)
{
foreach (var container in containersResult.Results)
{
await ProcessContainerAsync(container);
}
}
token = containersResult?.ContinuationToken;
if (token == null)
{
break;
}
}
if (_forcedUpdate)
{
_forcedUpdate = false;
}
_log.Info("Dwh structure update is finished");
}
public async Task <CloudContainersModel> ListContainers()
{
CloudContainersModel containersList = new CloudContainersModel();
try
{
var member = User.Identity.Name;
MemberLicense ml = this.iMemberLicenseRepository.MyLicense(member);
if (CloudStorageAccount.TryParse(ml?.AzureSaConnectionString, out CloudStorageAccount storageAccount))
{
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
BlobContinuationToken continuationToken = null;
do
{
ContainerResultSegment resultSegment = await blobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = resultSegment.ContinuationToken;
containersList.AddRange(resultSegment.Results);
}while (continuationToken != null);
}
}
catch
{
}
return(containersList);
}
private async Task ClearOldContainers(CloudBlobClient client)
{
var token = new BlobContinuationToken();
var containers = await client.ListContainersSegmentedAsync(ContainerPrefix, token)
.ConfigureAwait(false);
var validContainerNames = GetContainerNames();
if (containers?.Results == null)
{
return;
}
// loop through all containers
var results = containers.Results.ToList();
foreach (var container in results)
{
if (container.Name.StartsWith(ContainerPrefix, StringComparison.OrdinalIgnoreCase))
{
if (validContainerNames.NextName != container.Name &&
validContainerNames.CurrentName != container.Name &&
validContainerNames.PreviousName != container.Name)
{
// Delete the container if it doesn't match the current container or padding windows
await container.DeleteAsync().ConfigureAwait(false);
}
}
}
}
private static async Task DownloadFilesAsync()
{
CloudBlobClient blobClient = GetCloudBlobClient();
// Retrieve the list of containers in the storage account. Create a directory and configure variables for use later.
BlobContinuationToken continuationToken = null;
List <CloudBlobContainer> containers = new List <CloudBlobContainer>();
do
{
var listingResult = await blobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = listingResult.ContinuationToken;
containers.AddRange(listingResult.Results);
}while (continuationToken != null);
var directory = Directory.CreateDirectory("download");
BlobResultSegment resultSegment = null;
Stopwatch time = Stopwatch.StartNew();
// download thee blob
try
{
List <Task> Tasks = new List <Task>();
// Iterate throung the containers
foreach (CloudBlobContainer container in containers)
{
do
{
// Return the blobs from the container lazily 10 at a time.
resultSegment = await container.ListBlobsSegmentedAsync("", true, BlobListingDetails.All, 10, continuationToken, null, null);
{
foreach (var blobItem in resultSegment.Results)
{
if (((CloudBlob)blobItem).Properties.BlobType == BlobType.BlockBlob)
{
// Get the blob and add a task to download the blob asynchronously from the storage account.
CloudBlockBlob blockBlob = container.GetBlockBlobReference(((CloudBlockBlob)blobItem).Name);
Console.WriteLine("Starting download of {0} from container {1}", blockBlob.Name, container.Name);
Tasks.Add(blockBlob.DownloadToFileAsync(directory.FullName + "\\" + blockBlob.Name, FileMode.Create, null, new BlobRequestOptions()
{
DisableContentMD5Validation = true, StoreBlobContentMD5 = false
}, null));
}
}
}
}while (continuationToken != null);
}
// Creates an asynchonous task that completes when all the downloads complete.
await Task.WhenAll(Tasks);
}
catch (Exception e)
{
Console.WriteLine("\nThe transfer is canceled: {0}", e.Message);
}
time.Stop();
Console.WriteLine("Download has been completed in {0} seconds. Press any key to continue", time.Elapsed.TotalSeconds.ToString());
Console.ReadLine();
}
public void Dispose()
{
CloudBlobClient blobClient = _storageAccount.CreateCloudBlobClient();
foreach (var testContainer in blobClient.ListContainersSegmentedAsync(TestArtifactPrefix, null).Result.Results)
{
testContainer.DeleteAsync();
}
}
private static async Task <CloudBlobContainer> GetContainerAsync(CloudBlobClient cloudBlobClient)
{
var containers = new List <CloudBlobContainer>();
ContainerResultSegment response = await cloudBlobClient.ListContainersSegmentedAsync(null);
containers.AddRange(response.Results);
return(containers.FirstOrDefault());
}
private async Task Clear(CloudBlobClient target, CancellationToken token)
{
var response = await target.ListContainersSegmentedAsync(null);
do
{
foreach (var sourceItem in response.Results.Where(x => string.IsNullOrWhiteSpace(Options.Object) || x.Name.EqualsCi(Options.Object)).Where(x => !x.Name.StartsWith("azure-")))
{
var targetItem = target.GetContainerReference(sourceItem.Name);
_logger.LogInformation($"Deleting {sourceItem.GetType().Name} '{sourceItem.Name}'...");
await targetItem.DeleteAsync();
}
response = await target.ListContainersSegmentedAsync(response.ContinuationToken);
}while (response.ContinuationToken != null && !token.IsCancellationRequested);
}
static async Task ListarContainers(CloudBlobClient blobClient)
{
var containers = await blobClient.ListContainersSegmentedAsync(null);
foreach (var item in containers.Results)
{
Console.WriteLine(item.Name);
}
}
static async Task ListaContainers(CloudBlobClient blobClient)
{
//listar containers
var containers = await blobClient.ListContainersSegmentedAsync(null);
//retorna paginado entao nesse caso poderia passar um novo token para retornar as demais informações
foreach (var container in containers.Results)
{
System.Console.WriteLine(container.Name);
}
}
protected override async Task <LoadMoreResult> LoadMoreAsync(CollectionQuery query, object continuationToken)
{
var next = await client.ListContainersSegmentedAsync(continuationToken as BlobContinuationToken);
var result = new LoadMoreResult(next.Results.Where
(r =>
query.Filter == null ||
(r.Name.IndexOf(query.Filter, StringComparison.InvariantCultureIgnoreCase) >= 0)
).Select(r => new ContainerRecord(r) as object).ToList(), next.ContinuationToken);
return(result);
}
private async Task <List <CloudBlobContainer> > GetCloudBlobContainersAsync(CancellationToken cancellationToken)
{
var result = new List <CloudBlobContainer>();
ContainerResultSegment firstPage = await _client.ListContainersSegmentedAsync(null);
result.AddRange(firstPage.Results);
//todo: list more containers
return(result);
}
public void Dispose()
{
Host.Stop();
VerifyLockState("WebJobs.Internal.Blobs.Listener", LeaseState.Available, LeaseStatus.Unlocked).Wait();
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
foreach (var testContainer in blobClient.ListContainersSegmentedAsync(TestArtifactPrefix, null).Result.Results)
{
testContainer.DeleteAsync().Wait();
}
}
public async Task <bool> IsAvailableAsync()
{
try
{
BlobContinuationToken continuationToken = null;
await blobClient.ListContainersSegmentedAsync(continuationToken);
return(true);
}
catch
{
return(false);
}
}
private async Task <IEnumerable <CloudBlobContainer> > ListContainersAsync()
{
BlobContinuationToken continuationToken = null;
var response = new List <CloudBlobContainer>();
do
{
var segment = await CloudBlobClient.ListContainersSegmentedAsync(continuationToken);
continuationToken = segment.ContinuationToken;
response.AddRange(segment.Results);
}while (continuationToken != null);
return(response.Where(o => o.Properties.PublicAccess == BlobContainerPublicAccessType.Container));
}
// list containers in storage account
public IEnumerable <CloudBlobContainer> ListContainers()
{
var blobContainers = new List <CloudBlobContainer>();
BlobContinuationToken blobContinuationToken = null;
do
{
var containerSegment = _blobClient.ListContainersSegmentedAsync(blobContinuationToken).GetAwaiter().GetResult();
blobContainers.AddRange(containerSegment.Results);
blobContinuationToken = containerSegment.ContinuationToken;
} while (blobContinuationToken != null);
return(blobContainers);
}
public async Task <IViewComponentResult> InvokeAsync()
{
CloudStorageAccount cloudStorageAccount = _storageUtility.Value.StorageAccount;
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
var currentToken = new BlobContinuationToken();
List <string> categoriesContainerList = new List <string>();
var result = await cloudBlobClient.ListContainersSegmentedAsync(currentToken);
if (result.Results.Count() != 0)
{
categoriesContainerList = result.Results.Select(c => c.Name).ToList();
}
return(await Task.FromResult <IViewComponentResult>(View("Default", categoriesContainerList)));
}
public static async Task <List <CloudBlobContainer> > ListContainersAsync(this CloudBlobClient client)
{
BlobContinuationToken continuationToken = null;
List <CloudBlobContainer> results = new List <CloudBlobContainer>();
do
{
var response = await client.ListContainersSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}while (continuationToken != null);
return(results);
}
public async Task DisposeAsync()
{
await Host.StopAsync();
// $$$ reenalbe this
// VerifyLockState("WebJobs.Internal.Blobs.Listener", LeaseState.Available, LeaseStatus.Unlocked).Wait();
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
foreach (var testContainer in (await blobClient.ListContainersSegmentedAsync(TestArtifactPrefix, null)).Results)
{
await testContainer.DeleteAsync();
}
}
protected override async Task Execute()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
string blobPath = "";
Console.Clear();
Console.WriteLine("-----Add Blob-----");
Console.WriteLine($"Blob path:");
do
{
blobPath = Console.ReadLine().ToLower();
if (System.IO.File.Exists(blobPath))
{
Console.WriteLine();
Console.WriteLine("Select target container:");
ContainerResultSegment containers = await blobClient.ListContainersSegmentedAsync(null);
int counter = 1;
foreach (var container in containers.Results)
{
Console.WriteLine($"{counter++}) {container.Name}");
}
Console.Write("Selected container: ");
string itemNumberInput = Console.ReadLine();
if (int.TryParse(itemNumberInput, out int itemNumber) && itemNumber > 0 && itemNumber < containers.Results.Count() + 1)
{
var selectedContainer = containers.Results.ElementAt(itemNumber - 1);
CloudBlockBlob blob = selectedContainer.GetBlockBlobReference(Path.GetFileName(blobPath));
await blob.UploadFromFileAsync((blobPath));
Console.WriteLine($"Blob uri: {blob.Uri}");
await TextCopy.Clipboard.SetTextAsync(blob.Uri.ToString());
}
}
else
{
Console.WriteLine("File does not exist. Try again");
Console.SetCursorPosition(0, Console.CursorTop - 2);
}
} while (!System.IO.File.Exists(blobPath));
}
