private async Task ListFolderAsync(string path, List <Blob> container, ListOptions options) { IEnumerable <FileInfo> objects; try { objects = await _dbfs.List(StoragePath.Normalize(path)).ConfigureAwait(false); } catch (ClientApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound) { objects = new List <FileInfo>(); } List <Blob> batch = objects .Select(DConvert.ToBlob) .Where(options.IsMatch) .Where(b => options.BrowseFilter == null || options.BrowseFilter(b)) .ToList(); container.AddRange(batch); if (options.Recurse) { await Task.WhenAll(batch.Where(b => b.IsFolder).Select(f => ListFolderAsync(f.FullPath, container, options))).ConfigureAwait(false); } }
public async Task <IReadOnlyCollection <Blob> > ListFolderAsync(ListOptions options, CancellationToken cancellationToken) { var result = new List <Blob>(); await foreach (BlobHierarchyItem item in _client.GetBlobsByHierarchyAsync( delimiter: options.Recurse ? null : "/", prefix: FormatFolderPrefix(options.FolderPath), traits: options.IncludeAttributes ? BlobTraits.Metadata : BlobTraits.None).ConfigureAwait(false)) { Blob blob = AzConvert.ToBlob(_prependContainerName ? _client.Name : null, item); if (options.IsMatch(blob) && (options.BrowseFilter == null || options.BrowseFilter(blob))) { result.Add(blob); } } if (options.Recurse) { AssumeImplicitPrefixes( _prependContainerName ? StoragePath.Combine(_client.Name, options.FolderPath) : options.FolderPath, result); } return(result); }
public static async Task <IReadOnlyCollection <Blob> > ToBlobsAsync(PagedAsyncEnumerable <Objects, Object> pae, ListOptions options) { var result = new List <Blob>(); #pragma warning disable IDE0008 // Use explicit type (async enumerator conflict) using (var enumerator = pae.GetEnumerator()) #pragma warning restore IDE0008 // Use explicit type { while (await enumerator.MoveNext().ConfigureAwait(false)) { Object go = enumerator.Current; Blob blob = ToBlob(go); if (options.FilePrefix != null && !blob.Name.StartsWith(options.FilePrefix)) { continue; } if (options.BrowseFilter != null && !options.BrowseFilter(blob)) { continue; } result.Add(blob); if (options.MaxResults != null && result.Count >= options.MaxResults.Value) { break; } } } return(result); }
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken) { CloudBlobDirectory dir = GetCloudBlobDirectory(path); BlobContinuationToken token = null; var batch = new List <BlobId>(); await _throttler.WaitAsync(); try { do { BlobResultSegment segment = await dir.ListBlobsSegmentedAsync( false, BlobListingDetails.None, null, token, null, null, cancellationToken); token = segment.ContinuationToken; foreach (IListBlobItem blob in segment.Results) { BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown); if (options.IsMatch(id) && (options.BrowseFilter == null || options.BrowseFilter(id))) { batch.Add(id); } } }while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value))); } finally { _throttler.Release(); } batch = batch.Where(options.IsMatch).ToList(); if (options.Add(container, batch)) { return; } options.ListProgressCallback?.Invoke(container.Count, -1); if (options.Recurse) { List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList(); await Task.WhenAll( folderIds.Select(folderId => ListFolderAsync( container, StoragePath.Combine(path, folderId.Id), options, cancellationToken))); } }
private async Task <IReadOnlyCollection <Blob> > ListPathAsync(string path, int?maxResults, ListOptions options, CancellationToken cancellationToken) { //get filesystem name and folder path string[] parts = StoragePath.Split(path); string fs = parts[0]; string relativePath = StoragePath.Combine(parts.Skip(1)); var list = new List <Gen2Path>(); try { string continuation = null; do { string continuationParam = continuation == null ? null : $"&continuation={continuation}"; (PathList pl, IDictionary <string, string> responseHeaders) = await InvokeExtraAsync <PathList>( $"{fs}?resource=filesystem&directory={relativePath.UrlEncode()}&recursive={options.Recurse}{continuationParam}", RequestMethod.Get, cancellationToken).ConfigureAwait(false); list.AddRange(pl.Paths); responseHeaders.TryGetValue("x-ms-continuation", out continuation); } while(continuation != null); } catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound" || ex.ErrorCode == "FilesystemNotFound") { // trying to list a path which doesn't exist, just return an empty result return(new List <Blob>()); } IEnumerable <Blob> result = list.Select(p => AzConvert.ToBlob(fs, p)); if (options.FilePrefix != null) { result = result.Where(b => b.IsFolder || b.Name.StartsWith(options.FilePrefix)); } if (options.BrowseFilter != null) { result = result.Where(b => options.BrowseFilter(b)); } if (maxResults != null) { result = result.Take(maxResults.Value); } return(result.ToList()); }
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken) { var result = new List <Blob>(); var request = new ListObjectsV2Request() { BucketName = _bucketName, Prefix = options.FilePrefix ?? null }; while (options.MaxResults == null || (result.Count < options.MaxResults)) { ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken).ConfigureAwait(false); List <Blob> blobs = response.S3Objects .Where(s3Obj => !s3Obj.Key.EndsWith("/")) //these are "folders" in S3, but they don't always exist .Select(s3Obj => new Blob(StoragePath.RootFolderPath, s3Obj.Key, BlobItemKind.File) { Size = s3Obj.Size }) .Where(options.IsMatch) .Where(b => (options.FolderPath == null || StoragePath.ComparePath(b.FolderPath, options.FolderPath))) .Where(b => options.BrowseFilter == null || options.BrowseFilter(b)) .ToList(); result.AddRange(blobs); if (response.NextContinuationToken == null) { break; } request.ContinuationToken = response.NextContinuationToken; } if (options.MaxResults != null && result.Count > options.MaxResults) { result = result.Take((int)options.MaxResults).ToList(); } return(result); }
public static IEnumerable <Blob> ToBlobs(IEnumerable <Object> objects, ListOptions options) { foreach (Object obj in objects) { Blob item = ToBlob(obj); if (options.FilePrefix != null && !item.Name.StartsWith(options.FilePrefix)) { continue; } if (options.BrowseFilter != null && !options.BrowseFilter(item)) { continue; } yield return(item); } yield break; }
private async Task <IReadOnlyCollection <Blob> > ListFilesystemsAsync(ListOptions options, CancellationToken cancellationToken) { //todo: paging FilesystemList filesystems = await _api.ListFilesystemsAsync().ConfigureAwait(false); IEnumerable <Blob> result = filesystems.Filesystems .Select(LConvert.ToBlob); if (options.BrowseFilter != null) { result = result.Where(fs => options.BrowseFilter == null || options.BrowseFilter(fs)); } if (options.MaxResults != null) { result = result.Take(options.MaxResults.Value); } return(result.ToList()); }
private async Task <IReadOnlyCollection <Blob> > ListPathAsync(string path, int?maxResults, ListOptions options, CancellationToken cancellationToken) { //get filesystem name and folder path string[] parts = StoragePath.Split(path); string fs = parts[0]; string relativePath = StoragePath.Combine(parts.Skip(1)); PathList list; try { list = await _api.ListPathAsync(fs, relativePath, recursive : options.Recurse).ConfigureAwait(false); } catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound) { // specified path is not found, nothing serious return(new List <Blob>()); } IEnumerable <Blob> result = list.Paths.Select(p => LConvert.ToBlob(fs, p)); if (options.FilePrefix != null) { result = result.Where(b => b.IsFolder || b.Name.StartsWith(options.FilePrefix)); } if (options.BrowseFilter != null) { result = result.Where(b => options.BrowseFilter(b)); } if (maxResults != null) { result = result.Take(maxResults.Value); } return(result.ToList()); }
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken) { var result = new List <Blob>(); var request = new ListObjectsV2Request() { BucketName = _bucketName, Prefix = options.FilePrefix ?? null }; while (options.MaxResults == null || (result.Count < options.MaxResults)) { ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken).ConfigureAwait(false); List <Blob> blobs = response.S3Objects .Select(ToBlob) .Where(options.IsMatch) .Where(b => (options.FolderPath == null || StoragePath.ComparePath(b.FolderPath, options.FolderPath))) .Where(b => options.BrowseFilter == null || options.BrowseFilter(b)) .ToList(); result.AddRange(blobs); if (response.NextContinuationToken == null) { break; } request.ContinuationToken = response.NextContinuationToken; } if (options.MaxResults != null && result.Count > options.MaxResults) { result = result.Take((int)options.MaxResults).ToList(); } return(result); }
private static IReadOnlyCollection <Blob> ToBlobs(string path, ListObjectsV2Response response, ListOptions options) { var result = new List <Blob>(); //the files are listed as the S3Objects member, but they don't specifically contain folders, //but even if they do, they need to be filtered out result.AddRange( response.S3Objects .Where(b => !b.Key.EndsWith("/")) //check if this is "virtual folder" as S3 console creates them (rubbish) .Select(ToBlob) .Where(options.IsMatch) .Where(b => options.BrowseFilter == null || options.BrowseFilter(b))); //subfolders are listed in another field (what a funny name!) //prefix is absolute too result.AddRange( response.CommonPrefixes .Select(p => new Blob(p, BlobItemKind.Folder))); return(result); }
private async Task ListFolderAsync(List <Blob> container, string path, ListOptions options, CancellationToken cancellationToken) { CloudBlobDirectory dir = GetCloudBlobDirectory(path); BlobContinuationToken token = null; var batch = new List <Blob>(); await _throttler.WaitAsync(); try { do { BlobResultSegment segment = await dir.ListBlobsSegmentedAsync( false, //automatically include metadata in the response options.IncludeAttributes?BlobListingDetails.Metadata : BlobListingDetails.None, null, token, null, null, cancellationToken).ConfigureAwait(false); token = segment.ContinuationToken; foreach (IListBlobItem listItem in segment.Results) { Blob blob = ToBlob(listItem); if (options.IsMatch(blob) && (options.BrowseFilter == null || options.BrowseFilter(blob))) { batch.Add(blob); } } }while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value))); } finally { _throttler.Release(); } batch = batch.Where(options.IsMatch).ToList(); if (options.Add(container, batch)) { return; } if (options.Recurse) { var folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList(); await Task.WhenAll( folderIds.Select(folderId => ListFolderAsync( container, StoragePath.Combine(path, folderId.Name), options, cancellationToken))).ConfigureAwait(false); } }
public static IReadOnlyCollection <Blob> ToBlobs( this ListObjectsV2Response response, ListOptions options) { List <Blob> blobList = new List <Blob>(); blobList.AddRange(response.S3Objects.Where(b => !b.Key.EndsWith("/")).Select(b => b.ToBlob()).Where(options.IsMatch).Where(b => options.BrowseFilter == null || options.BrowseFilter(b))); blobList.AddRange(response.CommonPrefixes.Select(p => new Blob(p, BlobItemKind.Folder))); return(blobList); }