/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IEnumerable <IStorageBlob> > PollNewBlobsAsync(
IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken)
{
IEnumerable <IStorageListBlobItem> currentBlobs;
IStorageBlobResultSegment blobSegment;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken;
// if starting the cycle, keep the current time stamp to be used as curser
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow;
}
try
{
blobSegment = await container.ListBlobsSegmentedAsync(prefix : null, useFlatBlobListing : true,
blobListingDetails : BlobListingDetails.None, maxResults : blobPollLimitPerContainer, currentToken : continuationToken,
options : null, operationContext : null, cancellationToken : cancellationToken);
currentBlobs = blobSegment.Results;
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return(Enumerable.Empty <IStorageBlob>());
}
else
{
throw;
}
}
List <IStorageBlob> newBlobs = new List <IStorageBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
IStorageBlobProperties properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime)
{
newBlobs.Add(currentBlob);
}
}
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = blobSegment.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime
if (blobSegment.ContinuationToken == null)
{
containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime;
}
return(newBlobs);
}
public static async Task <IEnumerable <IStorageListBlobItem> > ListBlobsAsync(this IStorageBlobContainer container,
bool useFlatBlobListing, CancellationToken cancellationToken)
{
if (container == null)
{
throw new ArgumentNullException("container");
}
List <IStorageListBlobItem> allResults = new List <IStorageListBlobItem>();
BlobContinuationToken currentToken = null;
IStorageBlobResultSegment result;
do
{
result = await container.ListBlobsSegmentedAsync(prefix : null, useFlatBlobListing : useFlatBlobListing,
blobListingDetails : BlobListingDetails.None, maxResults : null, currentToken : currentToken,
options : null, operationContext : null, cancellationToken : cancellationToken);
if (result != null)
{
IEnumerable <IStorageListBlobItem> currentResults = result.Results;
if (currentResults != null)
{
allResults.AddRange(currentResults);
}
}
}while (result != null && result.ContinuationToken != null);
return(allResults);
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task<IEnumerable<IStorageBlob>> PollNewBlobsAsync(
IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken)
{
IEnumerable<IStorageListBlobItem> currentBlobs;
IStorageBlobResultSegment blobSegment;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken;
// if starting the cycle, keep the current time stamp to be used as curser
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow;
}
try
{
blobSegment = await container.ListBlobsSegmentedAsync(prefix: null, useFlatBlobListing: true,
blobListingDetails: BlobListingDetails.None, maxResults: blobPollLimitPerContainer, currentToken: continuationToken,
options: null, operationContext: null, cancellationToken: cancellationToken);
currentBlobs = blobSegment.Results;
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return Enumerable.Empty<IStorageBlob>();
}
else
{
throw;
}
}
List<IStorageBlob> newBlobs = new List<IStorageBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
IStorageBlobProperties properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime)
{
newBlobs.Add(currentBlob);
}
}
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = blobSegment.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime
if (blobSegment.ContinuationToken == null)
{
containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime;
}
return newBlobs;
}