private async Task ListFolderAsync(string path, List <Blob> container, ListOptions options)
{
IEnumerable <FileInfo> objects;
try
{
objects = await _dbfs.List(StoragePath.Normalize(path)).ConfigureAwait(false);
}
catch (ClientApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
objects = new List <FileInfo>();
}
List <Blob> batch = objects
.Select(DConvert.ToBlob)
.Where(options.IsMatch)
.Where(b => options.BrowseFilter == null || options.BrowseFilter(b))
.ToList();
container.AddRange(batch);
if (options.Recurse)
{
await Task.WhenAll(batch.Where(b => b.IsFolder).Select(f => ListFolderAsync(f.FullPath, container, options))).ConfigureAwait(false);
}
}
public async Task <IReadOnlyCollection <Blob> > ListFolderAsync(ListOptions options, CancellationToken cancellationToken)
{
var result = new List <Blob>();
await foreach (BlobHierarchyItem item in
_client.GetBlobsByHierarchyAsync(
delimiter: options.Recurse ? null : "/",
prefix: FormatFolderPrefix(options.FolderPath),
traits: options.IncludeAttributes ? BlobTraits.Metadata : BlobTraits.None).ConfigureAwait(false))
{
Blob blob = AzConvert.ToBlob(_prependContainerName ? _client.Name : null, item);
if (options.IsMatch(blob) && (options.BrowseFilter == null || options.BrowseFilter(blob)))
{
result.Add(blob);
}
}
if (options.Recurse)
{
AssumeImplicitPrefixes(
_prependContainerName ? StoragePath.Combine(_client.Name, options.FolderPath) : options.FolderPath,
result);
}
return(result);
}
public static async Task <IReadOnlyCollection <Blob> > ToBlobsAsync(PagedAsyncEnumerable <Objects, Object> pae, ListOptions options)
{
var result = new List <Blob>();
#pragma warning disable IDE0008 // Use explicit type (async enumerator conflict)
using (var enumerator = pae.GetEnumerator())
#pragma warning restore IDE0008 // Use explicit type
{
while (await enumerator.MoveNext().ConfigureAwait(false))
{
Object go = enumerator.Current;
Blob blob = ToBlob(go);
if (options.FilePrefix != null && !blob.Name.StartsWith(options.FilePrefix))
{
continue;
}
if (options.BrowseFilter != null && !options.BrowseFilter(blob))
{
continue;
}
result.Add(blob);
if (options.MaxResults != null && result.Count >= options.MaxResults.Value)
{
break;
}
}
}
return(result);
}
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
await _throttler.WaitAsync();
try
{
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown);
if (options.IsMatch(id) && (options.BrowseFilter == null || options.BrowseFilter(id)))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
}
finally
{
_throttler.Release();
}
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
options.ListProgressCallback?.Invoke(container.Count, -1);
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
await Task.WhenAll(
folderIds.Select(folderId => ListFolderAsync(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken)));
}
}
private async Task <IReadOnlyCollection <Blob> > ListPathAsync(string path, int?maxResults, ListOptions options, CancellationToken cancellationToken)
{
//get filesystem name and folder path
string[] parts = StoragePath.Split(path);
string fs = parts[0];
string relativePath = StoragePath.Combine(parts.Skip(1));
var list = new List <Gen2Path>();
try
{
string continuation = null;
do
{
string continuationParam = continuation == null ? null : $"&continuation={continuation}";
(PathList pl, IDictionary <string, string> responseHeaders) = await InvokeExtraAsync <PathList>(
$"{fs}?resource=filesystem&directory={relativePath.UrlEncode()}&recursive={options.Recurse}{continuationParam}",
RequestMethod.Get,
cancellationToken).ConfigureAwait(false);
list.AddRange(pl.Paths);
responseHeaders.TryGetValue("x-ms-continuation", out continuation);
} while(continuation != null);
}
catch (RequestFailedException ex) when(ex.ErrorCode == "PathNotFound" || ex.ErrorCode == "FilesystemNotFound")
{
// trying to list a path which doesn't exist, just return an empty result
return(new List <Blob>());
}
IEnumerable <Blob> result = list.Select(p => AzConvert.ToBlob(fs, p));
if (options.FilePrefix != null)
{
result = result.Where(b => b.IsFolder || b.Name.StartsWith(options.FilePrefix));
}
if (options.BrowseFilter != null)
{
result = result.Where(b => options.BrowseFilter(b));
}
if (maxResults != null)
{
result = result.Take(maxResults.Value);
}
return(result.ToList());
}
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken)
{
var result = new List <Blob>();
var request = new ListObjectsV2Request()
{
BucketName = _bucketName,
Prefix = options.FilePrefix ?? null
};
while (options.MaxResults == null || (result.Count < options.MaxResults))
{
ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken).ConfigureAwait(false);
List <Blob> blobs = response.S3Objects
.Where(s3Obj => !s3Obj.Key.EndsWith("/")) //these are "folders" in S3, but they don't always exist
.Select(s3Obj => new Blob(StoragePath.RootFolderPath, s3Obj.Key, BlobItemKind.File)
{
Size = s3Obj.Size
})
.Where(options.IsMatch)
.Where(b => (options.FolderPath == null || StoragePath.ComparePath(b.FolderPath, options.FolderPath)))
.Where(b => options.BrowseFilter == null || options.BrowseFilter(b))
.ToList();
result.AddRange(blobs);
if (response.NextContinuationToken == null)
{
break;
}
request.ContinuationToken = response.NextContinuationToken;
}
if (options.MaxResults != null && result.Count > options.MaxResults)
{
result = result.Take((int)options.MaxResults).ToList();
}
return(result);
}
public static IEnumerable <Blob> ToBlobs(IEnumerable <Object> objects, ListOptions options)
{
foreach (Object obj in objects)
{
Blob item = ToBlob(obj);
if (options.FilePrefix != null && !item.Name.StartsWith(options.FilePrefix))
{
continue;
}
if (options.BrowseFilter != null && !options.BrowseFilter(item))
{
continue;
}
yield return(item);
}
yield break;
}
private async Task <IReadOnlyCollection <Blob> > ListFilesystemsAsync(ListOptions options, CancellationToken cancellationToken)
{
//todo: paging
FilesystemList filesystems = await _api.ListFilesystemsAsync().ConfigureAwait(false);
IEnumerable <Blob> result = filesystems.Filesystems
.Select(LConvert.ToBlob);
if (options.BrowseFilter != null)
{
result = result.Where(fs => options.BrowseFilter == null || options.BrowseFilter(fs));
}
if (options.MaxResults != null)
{
result = result.Take(options.MaxResults.Value);
}
return(result.ToList());
}
private async Task <IReadOnlyCollection <Blob> > ListPathAsync(string path, int?maxResults, ListOptions options, CancellationToken cancellationToken)
{
//get filesystem name and folder path
string[] parts = StoragePath.Split(path);
string fs = parts[0];
string relativePath = StoragePath.Combine(parts.Skip(1));
PathList list;
try
{
list = await _api.ListPathAsync(fs, relativePath, recursive : options.Recurse).ConfigureAwait(false);
}
catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
// specified path is not found, nothing serious
return(new List <Blob>());
}
IEnumerable <Blob> result = list.Paths.Select(p => LConvert.ToBlob(fs, p));
if (options.FilePrefix != null)
{
result = result.Where(b => b.IsFolder || b.Name.StartsWith(options.FilePrefix));
}
if (options.BrowseFilter != null)
{
result = result.Where(b => options.BrowseFilter(b));
}
if (maxResults != null)
{
result = result.Take(maxResults.Value);
}
return(result.ToList());
}
public async Task <IReadOnlyCollection <Blob> > ListAsync(ListOptions options, CancellationToken cancellationToken)
{
var result = new List <Blob>();
var request = new ListObjectsV2Request()
{
BucketName = _bucketName,
Prefix = options.FilePrefix ?? null
};
while (options.MaxResults == null || (result.Count < options.MaxResults))
{
ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken).ConfigureAwait(false);
List <Blob> blobs = response.S3Objects
.Select(ToBlob)
.Where(options.IsMatch)
.Where(b => (options.FolderPath == null || StoragePath.ComparePath(b.FolderPath, options.FolderPath)))
.Where(b => options.BrowseFilter == null || options.BrowseFilter(b))
.ToList();
result.AddRange(blobs);
if (response.NextContinuationToken == null)
{
break;
}
request.ContinuationToken = response.NextContinuationToken;
}
if (options.MaxResults != null && result.Count > options.MaxResults)
{
result = result.Take((int)options.MaxResults).ToList();
}
return(result);
}
private static IReadOnlyCollection <Blob> ToBlobs(string path, ListObjectsV2Response response, ListOptions options)
{
var result = new List <Blob>();
//the files are listed as the S3Objects member, but they don't specifically contain folders,
//but even if they do, they need to be filtered out
result.AddRange(
response.S3Objects
.Where(b => !b.Key.EndsWith("/")) //check if this is "virtual folder" as S3 console creates them (rubbish)
.Select(ToBlob)
.Where(options.IsMatch)
.Where(b => options.BrowseFilter == null || options.BrowseFilter(b)));
//subfolders are listed in another field (what a funny name!)
//prefix is absolute too
result.AddRange(
response.CommonPrefixes
.Select(p => new Blob(p, BlobItemKind.Folder)));
return(result);
}
private async Task ListFolderAsync(List <Blob> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <Blob>();
await _throttler.WaitAsync();
try
{
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false,
//automatically include metadata in the response
options.IncludeAttributes?BlobListingDetails.Metadata : BlobListingDetails.None,
null, token, null, null, cancellationToken).ConfigureAwait(false);
token = segment.ContinuationToken;
foreach (IListBlobItem listItem in segment.Results)
{
Blob blob = ToBlob(listItem);
if (options.IsMatch(blob) && (options.BrowseFilter == null || options.BrowseFilter(blob)))
{
batch.Add(blob);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
}
finally
{
_throttler.Release();
}
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
var folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
await Task.WhenAll(
folderIds.Select(folderId => ListFolderAsync(
container,
StoragePath.Combine(path, folderId.Name),
options,
cancellationToken))).ConfigureAwait(false);
}
}
public static IReadOnlyCollection <Blob> ToBlobs(
this ListObjectsV2Response response,
ListOptions options)
{
List <Blob> blobList = new List <Blob>();
blobList.AddRange(response.S3Objects.Where(b => !b.Key.EndsWith("/")).Select(b => b.ToBlob()).Where(options.IsMatch).Where(b => options.BrowseFilter == null || options.BrowseFilter(b)));
blobList.AddRange(response.CommonPrefixes.Select(p => new Blob(p, BlobItemKind.Folder)));
return(blobList);
}
