/// <summary>
/// Convert the details into ListBlobsIncludeItem values.
/// </summary>
/// <returns>ListBlobsIncludeItem values</returns>
internal static IEnumerable <ListBlobsIncludeItem> AsIncludeItems(BlobTraits traits, BlobStates states)
{
// NOTE: Multiple strings MUST be appended in alphabetic order or signing the string for authentication fails!
// TODO: Remove this requirement by pushing it closer to header generation.
var items = new List <ListBlobsIncludeItem>();
if ((traits & BlobTraits.CopyStatus) == BlobTraits.CopyStatus)
{
items.Add(ListBlobsIncludeItem.Copy);
}
if ((states & BlobStates.Deleted) == BlobStates.Deleted)
{
items.Add(ListBlobsIncludeItem.Deleted);
}
if ((traits & BlobTraits.Metadata) == BlobTraits.Metadata)
{
items.Add(ListBlobsIncludeItem.Metadata);
}
if ((states & BlobStates.Snapshots) == BlobStates.Snapshots)
{
items.Add(ListBlobsIncludeItem.Snapshots);
}
if ((states & BlobStates.Uncommitted) == BlobStates.Uncommitted)
{
items.Add(ListBlobsIncludeItem.Uncommittedblobs);
}
return(items.Count > 0 ? items : null);
}
IEnumerable <BlobClient> GetAllBlobs(BlobContainerClient container, BlobTraits traits, string prefix)
{
foreach (BlobItem blob in container.GetBlobs(traits, this.blobStates, prefix))
{
yield return(container.GetBlobClient(blob.Name));
}
}
static void PrintBlobsInContainer(BlobContainerClient container, BlobTraits blobListingDetails)
{
foreach (BlobHierarchyItem blob in container.GetBlobsByHierarchy(blobListingDetails, BlobStates.All))
{
Console.WriteLine("- {0} (is soft deleted: {1}, is snapshot: {2})", blob.Blob.Name, blob.Blob.Deleted, String.IsNullOrEmpty(blob.Blob.Snapshot) ? false : true);
}
}
public override AsyncPageable <BlobItem> GetBlobsAsync(BlobTraits traits = BlobTraits.None, BlobStates states = BlobStates.None, string prefix = null, CancellationToken cancellationToken = default)
{
if (GetBlobsAsyncException != null)
{
throw GetBlobsAsyncException;
}
return(new MockAsyncPageable <BlobItem>(Blobs));
}
public override AsyncPageable <BlobItem> GetBlobsAsync(
BlobTraits traits = BlobTraits.None,
BlobStates states = BlobStates.None,
string?prefix = null,
CancellationToken cancellationToken = default)
{
return(new BlobsList(StoredFiles.Keys));
}
public override AsyncPageable <BlobItem> GetBlobsAsync(BlobTraits traits = BlobTraits.None, BlobStates states = BlobStates.None, string prefix = null, CancellationToken cancellationToken = default)
{
if (GetBlobsAsyncException != null)
{
throw GetBlobsAsyncException;
}
return(new MockAsyncPageable <BlobItem>(Blobs.Where(b => prefix == null || b.Name.StartsWith(prefix, StringComparison.Ordinal))));
}
public GetBlobsAsyncCollection(
BlobContainerClient client,
BlobTraits traits,
BlobStates states,
string prefix)
{
_client = client;
_traits = traits;
_states = states;
_prefix = prefix;
}
public GetBlobsByHierarchyAsyncCollection(
BlobContainerClient client,
string delimiter,
BlobTraits traits,
BlobStates states,
string prefix)
{
_client = client;
_delimiter = delimiter;
_traits = traits;
_states = states;
_prefix = prefix;
}
/// <summary>
/// list blobs by blob prefix and container name
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="prefix">blob preifx</param>
/// <returns>An enumerable collection of IListBlobItem</returns>
internal async Task ListBlobsByPrefix(long taskId, IStorageBlobManagement localChannel, string containerName, string prefix, Func <string, bool> blobFilter = null, bool includeDeleted = false, bool includeVersion = false)
{
CloudBlobContainer container = await GetCloudBlobContainerByName(localChannel, containerName).ConfigureAwait(false);
BlobContainerClient track2container = AzureStorageContainer.GetTrack2BlobContainerClient(container, localChannel.StorageContext, ClientOptions);
int listCount = InternalMaxCount;
int MaxListCount = 5000;
int requestCount = MaxListCount;
int realListCount = 0;
BlobContinuationToken continuationToken = ContinuationToken;
string track2ContinuationToken = this.ContinuationToken is null ? null : this.ContinuationToken.NextMarker;
if (UseTrack2Sdk()) // For new feature only available on Track2 SDK, need list with Track2 SDK.
{
BlobTraits blobTraits = BlobTraits.Metadata | BlobTraits.CopyStatus; // | BlobTraits.Tags;
BlobStates blobStates = BlobStates.Snapshots;
if (includeDeleted)
{
blobStates = blobStates | BlobStates.Deleted;
}
if (includeVersion)
{
blobStates = blobStates | BlobStates.Version;
}
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
IEnumerator <Page <BlobItem> > enumerator = track2container.GetBlobs(blobTraits, blobStates, prefix, CmdletCancellationToken)
.AsPages(track2ContinuationToken, requestCount)
.GetEnumerator();
Page <BlobItem> page;
enumerator.MoveNext();
page = enumerator.Current;
foreach (BlobItem item in page.Values)
{
if (blobFilter == null || blobFilter(item.Name))
{
OutputStream.WriteObject(taskId, GetAzureStorageBlob(item, track2container, localChannel.StorageContext, page.ContinuationToken, ClientOptions));
}
realListCount++;
}
track2ContinuationToken = page.ContinuationToken;
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
} while (listCount > 0 && !string.IsNullOrEmpty(track2ContinuationToken));
}
else
{
BlobRequestOptions requestOptions = RequestOptions;
bool useFlatBlobListing = true;
BlobListingDetails details = BlobListingDetails.Snapshots | BlobListingDetails.Metadata | BlobListingDetails.Copy;
if (includeDeleted)
{
details = details | BlobListingDetails.Deleted;
}
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
BlobResultSegment blobResult = await localChannel.ListBlobsSegmentedAsync(container, prefix, useFlatBlobListing,
details, requestCount, continuationToken, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
foreach (IListBlobItem blobItem in blobResult.Results)
{
CloudBlob blob = blobItem as CloudBlob;
if (blob == null)
{
continue;
}
if (blobFilter == null || blobFilter(blob.Name))
{
WriteCloudBlobObject(taskId, localChannel, blob, blobResult.ContinuationToken);
realListCount++;
}
}
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
continuationToken = blobResult.ContinuationToken;
}while (listCount > 0 && continuationToken != null);
}
}
/// <summary>List files</summary>
public async Task <List <BlobItem> > ListFilesAsync(string prefix = null, BlobTraits traits = BlobTraits.None,
BlobStates states = BlobStates.None, CancellationToken continuationToken = default)
{
var result = new List <BlobItem>();
// For >= C# 8.0 use:
//await foreach (BlobItem blobItem in _blobContainerClient.GetBlobsAsync(traits, states, prefix, continuationToken))
//{
// result.Add(blobItem);
//}
// For < C# 8.0 use:
AsyncPageable <BlobItem> blobPages = _blobContainerClient.GetBlobsAsync(traits, states, prefix, continuationToken);
IAsyncEnumerator <BlobItem> enumerator = blobPages.GetAsyncEnumerator();
try
{
while (await enumerator.MoveNextAsync())
{
result.Add(enumerator.Current);
}
}
finally
{
await enumerator.DisposeAsync();
}
return(result);
}
public static List <BlobItem> GetAllBlobs(this BlobContainerClient containerClient, BlobTraits traits = BlobTraits.None | BlobTraits.Metadata)
{
List <BlobItem> blobs = new List <BlobItem>();
string continuationToken = null;
while (continuationToken != string.Empty)
{
var resultSegment = containerClient.GetBlobs(traits).AsPages(continuationToken);
foreach (Page <BlobItem> blobPage in resultSegment)
{
foreach (var blob in blobPage.Values)
{
blobs.Add(blob);
}
continuationToken = blobPage.ContinuationToken;
}
}
return(blobs);
}
static void AssertOptions(string expected, BlobTraits traits, BlobStates states) => Assert.AreEqual(
expected,
string.Join(",", System.Linq.Enumerable.Select(BlobExtensions.AsIncludeItems(traits, states), item => Azure.Storage.Blobs.BlobRestClient.Serialization.ToString(item))));
private IAsyncEnumerable <BlobItem> ListBlobs(BlobContainerClient container, string prefix, BlobTraits traits, BlobStates states)
{
// Clean up the prefix if required
prefix = prefix == null ? null : SafePath.MakeSafeFilePath(prefix);
return(container.GetBlobsAsync(traits, states, prefix));
}
