private async Task ListFolderAsync(List <Blob> container, string path, ListOptions options, CancellationToken cancellationToken) { CloudBlobDirectory dir = GetCloudBlobDirectory(path); BlobContinuationToken token = null; var batch = new List <Blob>(); await _throttler.WaitAsync(); try { do { BlobResultSegment segment = await dir.ListBlobsSegmentedAsync( false, //automatically include metadata in the response options.IncludeAttributes?BlobListingDetails.Metadata : BlobListingDetails.None, null, token, null, null, cancellationToken).ConfigureAwait(false); token = segment.ContinuationToken; foreach (IListBlobItem listItem in segment.Results) { Blob blob = ToBlob(listItem); if (options.IsMatch(blob) && (options.BrowseFilter == null || options.BrowseFilter(blob))) { batch.Add(blob); } } }while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value))); } finally { _throttler.Release(); } batch = batch.Where(options.IsMatch).ToList(); if (options.Add(container, batch)) { return; } if (options.Recurse) { var folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList(); await Task.WhenAll( folderIds.Select(folderId => ListFolderAsync( container, StoragePath.Combine(path, folderId.Name), options, cancellationToken))).ConfigureAwait(false); } }
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken) { CloudBlobDirectory dir = GetCloudBlobDirectory(path); BlobContinuationToken token = null; var batch = new List <BlobId>(); await _throttler.WaitAsync(); try { do { BlobResultSegment segment = await dir.ListBlobsSegmentedAsync( false, BlobListingDetails.None, null, token, null, null, cancellationToken); token = segment.ContinuationToken; foreach (IListBlobItem blob in segment.Results) { BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown); if (options.IsMatch(id) && (options.BrowseFilter == null || options.BrowseFilter(id))) { batch.Add(id); } } }while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value))); } finally { _throttler.Release(); } batch = batch.Where(options.IsMatch).ToList(); if (options.Add(container, batch)) { return; } options.ListProgressCallback?.Invoke(container.Count, -1); if (options.Recurse) { List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList(); await Task.WhenAll( folderIds.Select(folderId => ListFolderAsync( container, StoragePath.Combine(path, folderId.Id), options, cancellationToken))); } }
private async Task Browse(string path, ListOptions options, ICollection <BlobId> container, CancellationToken token) { List <BlobId> batch; try { /*IEnumerable<BlobId> entries = _client * .EnumerateDirectory(path, UserGroupRepresentation.ObjectID) * .Select(n => ToBlobId(path, n, options.IncludeMetaWhenKnown)) * .Where(options.IsMatch);*/ IEnumerable <BlobId> entries = (await EnumerateDirectoryAsync(path, options, UserGroupRepresentation.ObjectID)) .Where(options.IsMatch); if (options.BrowseFilter != null) { entries = entries.Where(options.BrowseFilter); } batch = entries.ToList(); } //skip files with forbidden access catch (AdlsException ex) when(ex.HttpStatus == HttpStatusCode.Forbidden || ex.HttpStatus == HttpStatusCode.NotFound) { batch = null; } if (batch == null) { return; } if (options.Add(container, batch)) { return; } if (options.Recurse) { List <BlobId> folders = batch.Where(b => b.Kind == BlobItemKind.Folder).ToList(); if (folders.Count > 0) { await Task.WhenAll( folders.Select(bid => Browse( StoragePath.Combine(path, bid.Id), options, container, token ))); } } }
private async Task BrowseAsync(string path, ListOptions options, ICollection <Blob> container, CancellationToken token) { List <Blob> batch; try { IEnumerable <Blob> entries = (await EnumerateDirectoryAsync(path, options, UserGroupRepresentation.ObjectID).ConfigureAwait(false)) .Where(options.IsMatch); if (options.BrowseFilter != null) { entries = entries.Where(options.BrowseFilter); } batch = entries.ToList(); } //skip files with forbidden access catch (AdlsException ex) when(ex.HttpStatus == HttpStatusCode.Forbidden || ex.HttpStatus == HttpStatusCode.NotFound) { batch = null; } if (batch == null) { return; } if (options.Add(container, batch)) { return; } if (options.Recurse) { var folders = batch.Where(b => b.Kind == BlobItemKind.Folder).ToList(); if (folders.Count > 0) { await Task.WhenAll( folders.Select(b => BrowseAsync( StoragePath.Combine(path, b.Name), options, container, token ))).ConfigureAwait(false); } } }
private async Task Browse(string path, ListOptions options, ICollection <BlobId> container, CancellationToken token) { FileStatusesResult statuses; try { statuses = await _client.FileSystem.ListFileStatusAsync(_accountName, path, null, null, null, null, token); } //skip files with forbidden access catch (AdlsErrorException ex) when(ex.Response.StatusCode == HttpStatusCode.Forbidden) { statuses = null; } if (statuses == null) { return; } List <BlobId> batch = statuses.FileStatuses.FileStatus .Select(p => ToBlobId(path, p)) .Where(options.IsMatch) .ToList(); if (options.Add(container, batch)) { return; } if (options.Recurse) { int folderCount = batch.Count(b => b.Kind == BlobItemKind.Folder); if (folderCount > 0) { await Task.WhenAll(batch.Select(bid => Browse( StoragePath.Combine(path, bid.Id), options, container, token ))); } } }
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken) { CloudBlobDirectory dir = GetCloudBlobDirectory(path); BlobContinuationToken token = null; var batch = new List <BlobId>(); do { BlobResultSegment segment = await dir.ListBlobsSegmentedAsync( false, BlobListingDetails.None, null, token, null, null, cancellationToken); token = segment.ContinuationToken; foreach (IListBlobItem blob in segment.Results) { BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown); if (options.IsMatch(id)) { batch.Add(id); } } }while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value))); batch = batch.Where(options.IsMatch).ToList(); if (options.Add(container, batch)) { return; } if (options.Recurse) { List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList(); foreach (BlobId folderId in folderIds) { await ListFolderAsync( container, StoragePath.Combine(path, folderId.Id), options, cancellationToken); } } }
public async Task ListFolder(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken) { CloudBlobDirectory dir = GetCloudBlobDirectory(path); BlobContinuationToken token = null; var batch = new List <BlobId>(); do { BlobResultSegment segment = await dir.ListBlobsSegmentedAsync( false, BlobListingDetails.None, null, token, null, null, cancellationToken); token = segment.ContinuationToken; foreach (IListBlobItem blob in segment.Results) { BlobId id; if (blob is CloudBlockBlob blockBlob) { id = new BlobId(blockBlob.Name, BlobItemKind.File); } else if (blob is CloudAppendBlob appendBlob) { id = new BlobId(appendBlob.Name, BlobItemKind.File); } else if (blob is CloudBlobDirectory dirBlob) { id = new BlobId(dirBlob.Prefix, BlobItemKind.Folder); } else { throw new InvalidOperationException($"unknown item type {blob.GetType()}"); } if (options.IsMatch(id)) { batch.Add(id); } } }while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value))); batch = batch.Where(options.IsMatch).ToList(); if (options.Add(container, batch)) { return; } if (options.Recurse) { List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList(); foreach (BlobId folderId in folderIds) { await ListFolder( container, StoragePath.Combine(path, folderId.Id), options, cancellationToken); } } }