public async Task RegisterAsync(BlobServiceClient blobServiceClient, BlobContainerClient container, ITriggerExecutor <BlobTriggerExecutorContext> triggerExecutor, CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ThrowIfDisposed();
// Register all in logPolling, there is no problem if we get 2 notifications of the new blob
await _pollLogStrategy.RegisterAsync(blobServiceClient, container, triggerExecutor, cancellationToken).ConfigureAwait(false);
if (!_scanInfo.TryGetValue(container, out ContainerScanInfo containerScanInfo))
{
// First, try to load serialized scanInfo for this container.
DateTime?latestStoredScan = await _blobScanInfoManager.LoadLatestScanAsync(blobServiceClient.AccountName, container.Name).ConfigureAwait(false);
containerScanInfo = new ContainerScanInfo()
{
Registrations = new List <ITriggerExecutor <BlobTriggerExecutorContext> >(),
LastSweepCycleLatestModified = latestStoredScan ?? DateTime.MinValue,
CurrentSweepCycleLatestModified = DateTime.MinValue,
ContinuationToken = null
};
Logger.InitializedScanInfo(_logger, container.Name, containerScanInfo.LastSweepCycleLatestModified);
_scanInfo.Add(container, containerScanInfo);
}
containerScanInfo.Registrations.Add(triggerExecutor);
}
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor, CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ThrowIfDisposed();
// Register all in logPolling, there is no problem if we get 2 notifications of the new blob
await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken);
ContainerScanInfo containerScanInfo;
if (!_scanInfo.TryGetValue(container, out containerScanInfo))
{
containerScanInfo = new ContainerScanInfo()
{
Registrations = new List <ITriggerExecutor <IStorageBlob> >(),
LastSweepCycleLatestModified = DateTime.MinValue,
CurrentSweepCycleLatestModified = DateTime.MinValue,
ContinuationToken = null
};
_scanInfo.Add(container, containerScanInfo);
}
containerScanInfo.Registrations.Add(triggerExecutor);
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IEnumerable <IStorageBlob> > PollNewBlobsAsync(
IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken)
{
IEnumerable <IStorageListBlobItem> currentBlobs;
IStorageBlobResultSegment blobSegment;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken;
// if starting the cycle, keep the current time stamp to be used as curser
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow;
}
try
{
blobSegment = await container.ListBlobsSegmentedAsync(prefix : null, useFlatBlobListing : true,
blobListingDetails : BlobListingDetails.None, maxResults : blobPollLimitPerContainer, currentToken : continuationToken,
options : null, operationContext : null, cancellationToken : cancellationToken);
currentBlobs = blobSegment.Results;
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return(Enumerable.Empty <IStorageBlob>());
}
else
{
throw;
}
}
List <IStorageBlob> newBlobs = new List <IStorageBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
IStorageBlobProperties properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime)
{
newBlobs.Add(currentBlob);
}
}
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = blobSegment.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime
if (blobSegment.ContinuationToken == null)
{
containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime;
}
return(newBlobs);
}
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken, List <IStorageBlob> failedNotifications)
{
cancellationToken.ThrowIfCancellationRequested();
DateTime lastScan = containerScanInfo.LastSweepCycleLatestModified;
IEnumerable <IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerScanInfo, cancellationToken);
foreach (IStorageBlob newBlob in newBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken);
}
// if the 'LatestModified' has changed, update it in the manager
if (containerScanInfo.LastSweepCycleLatestModified > lastScan)
{
DateTime latestScan = containerScanInfo.LastSweepCycleLatestModified;
// It's possible that we had some blobs that we failed to move to the queue. We want to make sure
// we continue to find these if the host needs to restart.
if (failedNotifications.Any())
{
latestScan = failedNotifications.Min(n => n.Properties.LastModified.Value.UtcDateTime);
}
// Store our timestamp slightly earlier than the last timestamp. This is a failsafe for any blobs that created
// milliseconds after our last scan (blob timestamps round to the second). This way we make sure to pick those
// up on a host restart.
await _blobScanInfoManager.UpdateLatestScanAsync(container.ServiceClient.Credentials.AccountName,
container.Name, latestScan.AddMilliseconds(-1));
}
}
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerInfo, CancellationToken cancellationToken, List <IStorageBlob> failedNotifications)
{
cancellationToken.ThrowIfCancellationRequested();
IEnumerable <IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerInfo, cancellationToken);
foreach (IStorageBlob newBlob in newBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken);
}
}
private async Task PollAndNotify(BlobContainerClient container, ContainerScanInfo containerScanInfo,
List <BlobNotification> failedNotifications, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
DateTime lastScan = containerScanInfo.LastSweepCycleLatestModified;
// For tracking
string clientRequestId = Guid.NewGuid().ToString();
IEnumerable <BlobBaseClient> newBlobs = await PollNewBlobsAsync(container, containerScanInfo, clientRequestId, cancellationToken).ConfigureAwait(false);
foreach (var newBlob in newBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
await NotifyRegistrationsAsync(new BlobWithContainer <BlobBaseClient>(container, newBlob), failedNotifications, clientRequestId, cancellationToken).ConfigureAwait(false);
}
// if the 'LatestModified' has changed, update it in the manager
if (containerScanInfo.LastSweepCycleLatestModified > lastScan)
{
DateTime latestScan = containerScanInfo.LastSweepCycleLatestModified;
// It's possible that we had some blobs that we failed to move to the queue. We want to make sure
// we continue to find these if the host needs to restart.
if (failedNotifications.Any())
{
// TODO (kasobol-msft) this call to GetProperties is suboptimal figure out how to propagate data from listing here.
latestScan = failedNotifications.Select(p => p.Blob).Min(n => n.BlobClient.GetProperties().Value.LastModified.UtcDateTime);
}
// Store our timestamp slightly earlier than the last timestamp. This is a failsafe for any blobs that created
// milliseconds after our last scan (blob timestamps round to the second). This way we make sure to pick those
// up on a host restart.
await _blobScanInfoManager.UpdateLatestScanAsync(container.AccountName,
container.Name, latestScan.AddMilliseconds(-1)).ConfigureAwait(false);
}
}
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor, CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ThrowIfDisposed();
// Register all in logPolling, there is no problem if we get 2 notifications of the new blob
await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken);
ContainerScanInfo containerScanInfo;
if (!_scanInfo.TryGetValue(container, out containerScanInfo))
{
containerScanInfo = new ContainerScanInfo()
{
Registrations = new List<ITriggerExecutor<IStorageBlob>>(),
LastSweepCycleStartTime = DateTime.MinValue,
CurrentSweepCycleStartTime = DateTime.MinValue,
ContinuationToken = null
};
_scanInfo.Add(container, containerScanInfo);
}
containerScanInfo.Registrations.Add(triggerExecutor);
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="clientRequestId"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IEnumerable <BlobBaseClient> > PollNewBlobsAsync(
BlobContainerClient container, ContainerScanInfo containerScanInfo, string clientRequestId, CancellationToken cancellationToken)
{
IEnumerable <BlobItem> currentBlobs;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
string continuationToken = containerScanInfo.ContinuationToken;
Page <BlobItem> page;
// if starting the cycle, reset the sweep time
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue;
}
Stopwatch sw = Stopwatch.StartNew();
try
{
AsyncPageable <BlobItem> blobsAsyncPageable = container.GetBlobsAsync(cancellationToken: cancellationToken);
IAsyncEnumerable <Page <BlobItem> > pages = blobsAsyncPageable.AsPages(continuationToken: continuationToken, pageSizeHint: blobPollLimitPerContainer);
IAsyncEnumerator <Page <BlobItem> > pagesEnumerator = pages.GetAsyncEnumerator(cancellationToken);
if (await pagesEnumerator.MoveNextAsync().ConfigureAwait(false))
{
page = pagesEnumerator.Current;
currentBlobs = page.Values;
}
else
{
return(Enumerable.Empty <BlobBaseClient>());
}
}
catch (RequestFailedException exception)
{
if (exception.IsNotFound())
{
Logger.ContainerDoesNotExist(_logger, container.Name);
return(Enumerable.Empty <BlobBaseClient>());
}
else
{
throw;
}
}
List <BlobBaseClient> newBlobs = new List <BlobBaseClient>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (BlobItem currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
var properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified)
{
containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp;
}
// Blob timestamps are rounded to the nearest second, so make sure we continue to check
// the previous timestamp to catch any blobs that came in slightly after our previous poll.
if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified)
{
newBlobs.Add(container.GetBlobClient(currentBlob.Name));
}
}
Logger.PollBlobContainer(_logger, container.Name, containerScanInfo.LastSweepCycleLatestModified, clientRequestId,
newBlobs.Count, sw.ElapsedMilliseconds, !string.IsNullOrWhiteSpace(page.ContinuationToken));
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = page.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime, if changed
if (page.ContinuationToken == null &&
containerScanInfo.CurrentSweepCycleLatestModified > containerScanInfo.LastSweepCycleLatestModified)
{
containerScanInfo.LastSweepCycleLatestModified = containerScanInfo.CurrentSweepCycleLatestModified;
}
return(newBlobs);
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task<IEnumerable<IStorageBlob>> PollNewBlobsAsync(
IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken)
{
IEnumerable<IStorageListBlobItem> currentBlobs;
IStorageBlobResultSegment blobSegment;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken;
// if starting the cycle, keep the current time stamp to be used as curser
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow;
}
try
{
blobSegment = await container.ListBlobsSegmentedAsync(prefix: null, useFlatBlobListing: true,
blobListingDetails: BlobListingDetails.None, maxResults: blobPollLimitPerContainer, currentToken: continuationToken,
options: null, operationContext: null, cancellationToken: cancellationToken);
currentBlobs = blobSegment.Results;
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return Enumerable.Empty<IStorageBlob>();
}
else
{
throw;
}
}
List<IStorageBlob> newBlobs = new List<IStorageBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
IStorageBlobProperties properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime)
{
newBlobs.Add(currentBlob);
}
}
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = blobSegment.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime
if (blobSegment.ContinuationToken == null)
{
containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime;
}
return newBlobs;
}
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerInfo, CancellationToken cancellationToken, List<IStorageBlob> failedNotifications)
{
cancellationToken.ThrowIfCancellationRequested();
IEnumerable<IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerInfo, cancellationToken);
foreach (IStorageBlob newBlob in newBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken);
}
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IEnumerable <ICloudBlob> > PollNewBlobsAsync(
CloudBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken)
{
IEnumerable <IListBlobItem> currentBlobs;
BlobResultSegment blobSegment;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken;
// if starting the cycle, reset the sweep time
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue;
}
try
{
blobSegment = await container.ListBlobsSegmentedAsync(prefix : null, useFlatBlobListing : true,
blobListingDetails : BlobListingDetails.None, maxResults : blobPollLimitPerContainer, currentToken : continuationToken,
options : null, operationContext : null, cancellationToken : cancellationToken);
currentBlobs = blobSegment.Results;
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return(Enumerable.Empty <ICloudBlob>());
}
else
{
throw;
}
}
List <ICloudBlob> newBlobs = new List <ICloudBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (ICloudBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
var properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified)
{
containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp;
}
// Blob timestamps are rounded to the nearest second, so make sure we continue to check
// the previous timestamp to catch any blobs that came in slightly after our previous poll.
if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified)
{
newBlobs.Add(currentBlob);
}
}
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = blobSegment.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime, if changed
if (blobSegment.ContinuationToken == null &&
containerScanInfo.CurrentSweepCycleLatestModified > containerScanInfo.LastSweepCycleLatestModified)
{
containerScanInfo.LastSweepCycleLatestModified = containerScanInfo.CurrentSweepCycleLatestModified;
}
return(newBlobs);
}
