/// <summary>
/// Gets the directory under the directory
/// </summary>
/// <param name="path">directory path</param>
/// <returns></returns>
public async Task <IEnumerable <CloudBlobDirectory> > GetDirectoryListing(string path)
{
// Get the full path of directory
string prefix = GetFullPath(path);
var results = await _blobClient.ListBlobsSegmentedAsync(prefix, null);
return(results.Results.OfType <CloudBlobDirectory>());
}
private static async Task ListBlobsFromServiceClientAsync(CloudBlobClient blobClient, string prefix)
{
Console.WriteLine("List blobs by prefix. Prefix must include container name:");
BlobContinuationToken continuationToken = null;
BlobResultSegment resultSegment = null;
try
{
do
{
// The prefix is required when listing blobs from the service client. The prefix must include
// the container name.
resultSegment = await blobClient.ListBlobsSegmentedAsync(prefix, continuationToken);
foreach (var blob in resultSegment.Results)
{
Console.WriteLine("\tBlob:" + blob.Uri);
}
Console.WriteLine();
// Get the continuation token.
continuationToken = resultSegment.ContinuationToken;
} while (continuationToken != null);
}
catch (StorageException e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
public static async Task <IEnumerable <IListBlobItem> > ListBlobsAsync(this CloudBlobClient client,
string prefix, bool useFlatBlobListing, BlobListingDetails blobListingDetails,
CancellationToken cancellationToken)
{
if (client == null)
{
throw new ArgumentNullException("client");
}
List <IListBlobItem> allResults = new List <IListBlobItem>();
BlobContinuationToken continuationToken = null;
BlobResultSegment result;
do
{
result = await client.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails,
maxResults : null, currentToken : continuationToken, options : null, operationContext : null
);
if (result != null)
{
IEnumerable <IListBlobItem> currentResults = result.Results;
if (currentResults != null)
{
allResults.AddRange(currentResults);
}
continuationToken = result.ContinuationToken;
}
}while (result != null && continuationToken != null);
return(allResults);
}
public async Task <DataResponse <List <Uri> > > List(DateTime day)
{
// Loop over items within the container and output the length and URI.
BlobContinuationToken continuationToken = null;
BlobResultSegment resultSegment = null;
DataResponse <List <Uri> > resp = new DataResponse <List <Uri> >();
resp.Data = new List <Uri>();
try
{
do
{
// The prefix is required when listing blobs from the service client. The prefix must include
// the container name.
resultSegment = await _blobClient.ListBlobsSegmentedAsync(day.ToString("HH-dd"), continuationToken);
foreach (var blob in resultSegment.Results)
{
resp.Data.Add(blob.Uri);
}
// Get the continuation token.
continuationToken = resultSegment.ContinuationToken;
} while (continuationToken != null);
}
catch (StorageException e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
return(resp);
}
internal async Task AddBlobsToContainer(DirectoryBlock directory)
{
var container = client.GetContainerReference(directory.Name);
var items = await container.ListBlobsSegmentedAsync(null, false, new BlobListingDetails(), maxListResults, new BlobContinuationToken(), null, null);
var list_continue = false;
do
{
foreach (var item in items.Results)
{
AddToParentDirectoryObject(directory, item);
}
if (items.ContinuationToken != null)
{
items = await client.ListBlobsSegmentedAsync(null, false, new BlobListingDetails(), maxListResults, items.ContinuationToken, null, null);
list_continue = true;
}
else
{
list_continue = false;
}
} while (list_continue);
}
/// <inheritdoc />
public Task <IStorageBlobResultSegment> ListBlobsSegmentedAsync(string prefix, bool useFlatBlobListing,
BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken,
BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
Task <BlobResultSegment> sdkTask = _sdk.ListBlobsSegmentedAsync(prefix, useFlatBlobListing,
blobListingDetails, maxResults, currentToken, options, operationContext);
return(ListBlobsSegmentedAsyncCore(sdkTask));
}
public BlobResultSegment ListBlobsSegmented(string prefix, bool useFlatListing, BlobListingDetails blobListingDetails,
int?maxResults, BlobContinuationToken continuationToken, BlobRequestOptions blobRequestOptions,
OperationContext operationContext)
{
var task = _client.ListBlobsSegmentedAsync(prefix, useFlatListing, blobListingDetails, maxResults, continuationToken,
blobRequestOptions, operationContext);
task.Wait();
return(task.Result);
}
public static async Task <List <IListBlobItem> > ListBlobsAsync(this CloudBlobClient client, string prefix)
{
BlobContinuationToken continuationToken = null;
List <IListBlobItem> results = new List <IListBlobItem>();
do
{
var response = await client.ListBlobsSegmentedAsync(prefix, continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}while (continuationToken != null);
return(results);
}
public static async System.Threading.Tasks.Task <IEnumerable <IListBlobItem> > ListBlobsAsync(this CloudBlobClient blobClient, string prefix)
{
BlobContinuationToken token = null;
var retVal = new List <IListBlobItem>();
do
{
var results = await blobClient.ListBlobsSegmentedAsync(prefix, token);
retVal.AddRange(results.Results);
token = results.ContinuationToken;
} while (token != null);
return(retVal);
}
/// <summary>
/// Returns the list of blobs matching the specified path.
/// </summary>
/// <param name="blobPathPrefix"></param>
/// <returns></returns>
public async Task <List <BlobInfo> > ListBlobAsync(string blobPathPrefix)
{
//$root container is not implicitly supported, at least an explicit container name is expected
string prefix = blobPathPrefix ?? throw new ArgumentNullException(nameof(blobPathPrefix));
if (string.IsNullOrWhiteSpace(prefix))
{
throw new ArgumentException(nameof(blobPathPrefix));
}
CloudBlobClient blobClient = m_LazyBlobClient.Value;
BlobContinuationToken continuationToken = null;
//pad prefix path for container only with trailing slash '/', otherwise storage SDK list in $root container only
if (!prefix.Contains('/'))
{
prefix += '/';
}
var result = new List <BlobInfo>();
try
{
do
{
BlobResultSegment segment = await blobClient.ListBlobsSegmentedAsync(prefix, true, BlobListingDetails.Metadata, null, continuationToken, null, null);
continuationToken = segment.ContinuationToken;
result.AddRange(segment.Results.Cast <CloudBlob>().Select(b => AzureBlob.FromBlobProperties(GetBlobPath(b), b.Properties, b.Metadata)));
} while (continuationToken != null);
}
catch (StorageException ex) when(ex.RequestInformation.ErrorCode == BlobErrorCodeStrings.ContainerNotFound)
{
//swallow container not found and let the default return the currently empty list instead
}
return(result);
}
private void DeleteOldBlobsFromContainer(CloudBlobClient blobClient, string containerName)
{
// Figure out the timestamp before which all blobs will be deleted
DateTime cutoffTime = DateTime.UtcNow.Add(-this.blobDeletionAge);
this.traceSource.WriteInfo(
this.logSourceId,
"Deleting blobs older than {0}.",
cutoffTime);
// List all blobs in the container that need to be deleted
//
// Notes:
// #1. The blobs have a virtual directory hierarchy inside the
// container, so we explicitly opt for a flat listing of blobs.
// #2. If the Azure runtime interfaces are available, then the top-
// level virtual directory we create is inside the container
// is the deployment ID. In this case, we only delete blobs under
// the virtual container whose deployment ID matches our current
// deployment ID.
string prefix = string.Format(
CultureInfo.InvariantCulture,
"{0}{1}{2}{3}{4}{5}{6}{7}",
containerName,
blobClient.DefaultDelimiter,
this.azureInterfaceAvailable ? (string.IsNullOrEmpty(this.deploymentId) ? AzureUtility.DeploymentId : this.deploymentId) : this.fabricNodeInstanceName,
blobClient.DefaultDelimiter,
this.azureInterfaceAvailable ? AzureUtility.RoleName : string.Empty,
this.azureInterfaceAvailable ? blobClient.DefaultDelimiter : string.Empty,
this.azureInterfaceAvailable ? AzureUtility.RoleInstanceId : string.Empty,
this.azureInterfaceAvailable ? blobClient.DefaultDelimiter : string.Empty);
BlobContinuationToken continuationToken = null;
BlobRequestOptions requestOptions = new BlobRequestOptions();
OperationContext operationContext = new OperationContext();
do
{
BlobResultSegment resultSegment;
try
{
this.perfHelper.ExternalOperationBegin(
ExternalOperationTime.ExternalOperationType.BlobQuery,
0);
// Get the blobs that were uploaded by the current node. The prefix
// helps us identify them.
#if DotNetCoreClr
resultSegment = blobClient.ListBlobsSegmentedAsync(
prefix,
true, // useFlatBlobListing
BlobListingDetails.All,
null,
continuationToken,
requestOptions,
operationContext).Result;
#else
resultSegment = blobClient.ListBlobsSegmented(
prefix,
true, // useFlatBlobListing
BlobListingDetails.All,
null,
continuationToken,
requestOptions,
operationContext);
#endif
this.perfHelper.ExternalOperationEnd(
ExternalOperationTime.ExternalOperationType.BlobQuery,
0);
}
catch (Exception e)
{
this.traceSource.WriteError(
this.logSourceId,
"Exception encountered when attempting to enumerate blobs for deletion in container {0} in storage account {1}. Exception information: {2}",
containerName,
this.storageAccountFactory.Connection.AccountName,
e);
// If we encounter an error during enumeration of blobs for deletion,
// we give up.
this.BlobQueryAndDeleteExceptionHandler(e);
break;
}
continuationToken = resultSegment.ContinuationToken;
if (!resultSegment.Results.Any())
{
continue;
}
this.perfHelper.AzureBlobsQueried((ulong)resultSegment.Results.Count());
// Get the blobs that are old enough to be deleted
IEnumerable <IListBlobItem> blobsToDelete = resultSegment.Results
.Where(blob => this.ShouldDeleteBlob(
blob,
cutoffTime));
// Go through the list and delete the blobs
foreach (IListBlobItem blobInterface in blobsToDelete)
{
ICloudBlob blob = (ICloudBlob)blobInterface;
try
{
this.perfHelper.ExternalOperationBegin(
ExternalOperationTime.ExternalOperationType.BlobDeletion,
0);
// DeleteIfExists allows for the case where the blob is being
// deleted by other means (possibly by another instance of
// the DCA) at the same time that we are trying to delete it.
#if DotNetCoreClr
blob.DeleteIfExistsAsync().Wait();
#else
blob.DeleteIfExists();
#endif
this.perfHelper.ExternalOperationEnd(
ExternalOperationTime.ExternalOperationType.BlobDeletion,
0);
this.perfHelper.AzureBlobDeleted();
}
catch (Exception e)
{
this.traceSource.WriteError(
this.logSourceId,
"Exception encountered when attempting to delete blob {0} in container {1} in storage account {2}. Exception information: {3}",
blob.Name,
containerName,
this.storageAccountFactory.Connection.AccountName,
e);
// If we encounter an error during the deletion of one blob,
// we'll still try and delete the others
this.BlobQueryAndDeleteExceptionHandler(e);
}
if (this.stopping)
{
this.traceSource.WriteInfo(
this.logSourceId,
"The consumer is being stopped. Therefore, no more old blobs will be deleted from container {0} in storage account {1}.",
containerName,
this.storageAccountFactory.Connection.AccountName);
break;
}
}
if (this.stopping)
{
this.traceSource.WriteInfo(
this.logSourceId,
"The consumer is being stopped. Therefore, no more blobs will be enumerated for deletion from container {0} in storage account {1}.",
containerName,
this.storageAccountFactory.Connection.AccountName);
break;
}
}while (continuationToken != null);
}
