private async Task ListFolderAsync(List <Blob> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <Blob>();
await _throttler.WaitAsync();
try
{
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false,
//automatically include metadata in the response
options.IncludeAttributes?BlobListingDetails.Metadata : BlobListingDetails.None,
null, token, null, null, cancellationToken).ConfigureAwait(false);
token = segment.ContinuationToken;
foreach (IListBlobItem listItem in segment.Results)
{
Blob blob = ToBlob(listItem);
if (options.IsMatch(blob) && (options.BrowseFilter == null || options.BrowseFilter(blob)))
{
batch.Add(blob);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
}
finally
{
_throttler.Release();
}
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
var folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
await Task.WhenAll(
folderIds.Select(folderId => ListFolderAsync(
container,
StoragePath.Combine(path, folderId.Name),
options,
cancellationToken))).ConfigureAwait(false);
}
}
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
await _throttler.WaitAsync();
try
{
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown);
if (options.IsMatch(id) && (options.BrowseFilter == null || options.BrowseFilter(id)))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
}
finally
{
_throttler.Release();
}
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
options.ListProgressCallback?.Invoke(container.Count, -1);
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
await Task.WhenAll(
folderIds.Select(folderId => ListFolderAsync(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken)));
}
}
private async Task Browse(string path, ListOptions options, ICollection <BlobId> container, CancellationToken token)
{
List <BlobId> batch;
try
{
/*IEnumerable<BlobId> entries = _client
* .EnumerateDirectory(path, UserGroupRepresentation.ObjectID)
* .Select(n => ToBlobId(path, n, options.IncludeMetaWhenKnown))
* .Where(options.IsMatch);*/
IEnumerable <BlobId> entries =
(await EnumerateDirectoryAsync(path, options, UserGroupRepresentation.ObjectID))
.Where(options.IsMatch);
if (options.BrowseFilter != null)
{
entries = entries.Where(options.BrowseFilter);
}
batch = entries.ToList();
}
//skip files with forbidden access
catch (AdlsException ex) when(ex.HttpStatus == HttpStatusCode.Forbidden || ex.HttpStatus == HttpStatusCode.NotFound)
{
batch = null;
}
if (batch == null)
{
return;
}
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
List <BlobId> folders = batch.Where(b => b.Kind == BlobItemKind.Folder).ToList();
if (folders.Count > 0)
{
await Task.WhenAll(
folders.Select(bid => Browse(
StoragePath.Combine(path, bid.Id),
options,
container,
token
)));
}
}
}
private async Task BrowseAsync(string path, ListOptions options, ICollection <Blob> container, CancellationToken token)
{
List <Blob> batch;
try
{
IEnumerable <Blob> entries =
(await EnumerateDirectoryAsync(path, options, UserGroupRepresentation.ObjectID).ConfigureAwait(false))
.Where(options.IsMatch);
if (options.BrowseFilter != null)
{
entries = entries.Where(options.BrowseFilter);
}
batch = entries.ToList();
}
//skip files with forbidden access
catch (AdlsException ex) when(ex.HttpStatus == HttpStatusCode.Forbidden || ex.HttpStatus == HttpStatusCode.NotFound)
{
batch = null;
}
if (batch == null)
{
return;
}
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
var folders = batch.Where(b => b.Kind == BlobItemKind.Folder).ToList();
if (folders.Count > 0)
{
await Task.WhenAll(
folders.Select(b => BrowseAsync(
StoragePath.Combine(path, b.Name),
options,
container,
token
))).ConfigureAwait(false);
}
}
}
private async Task Browse(string path, ListOptions options, ICollection <BlobId> container, CancellationToken token)
{
FileStatusesResult statuses;
try
{
statuses = await _client.FileSystem.ListFileStatusAsync(_accountName, path,
null, null, null, null,
token);
}
//skip files with forbidden access
catch (AdlsErrorException ex) when(ex.Response.StatusCode == HttpStatusCode.Forbidden)
{
statuses = null;
}
if (statuses == null)
{
return;
}
List <BlobId> batch =
statuses.FileStatuses.FileStatus
.Select(p => ToBlobId(path, p))
.Where(options.IsMatch)
.ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
int folderCount = batch.Count(b => b.Kind == BlobItemKind.Folder);
if (folderCount > 0)
{
await Task.WhenAll(batch.Select(bid => Browse(
StoragePath.Combine(path, bid.Id),
options,
container,
token
)));
}
}
}
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown);
if (options.IsMatch(id))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
foreach (BlobId folderId in folderIds)
{
await ListFolderAsync(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken);
}
}
}
public async Task ListFolder(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id;
if (blob is CloudBlockBlob blockBlob)
{
id = new BlobId(blockBlob.Name, BlobItemKind.File);
}
else if (blob is CloudAppendBlob appendBlob)
{
id = new BlobId(appendBlob.Name, BlobItemKind.File);
}
else if (blob is CloudBlobDirectory dirBlob)
{
id = new BlobId(dirBlob.Prefix, BlobItemKind.Folder);
}
else
{
throw new InvalidOperationException($"unknown item type {blob.GetType()}");
}
if (options.IsMatch(id))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
foreach (BlobId folderId in folderIds)
{
await ListFolder(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken);
}
}
}
