public async Task RegisterAsync(BlobServiceClient blobServiceClient, BlobContainerClient container, ITriggerExecutor <BlobTriggerExecutorContext> triggerExecutor, CancellationToken cancellationToken) { // Register and Execute are not concurrency-safe. // Avoiding calling Register while Execute is running is the caller's responsibility. ThrowIfDisposed(); // Register all in logPolling, there is no problem if we get 2 notifications of the new blob await _pollLogStrategy.RegisterAsync(blobServiceClient, container, triggerExecutor, cancellationToken).ConfigureAwait(false); if (!_scanInfo.TryGetValue(container, out ContainerScanInfo containerScanInfo)) { // First, try to load serialized scanInfo for this container. DateTime?latestStoredScan = await _blobScanInfoManager.LoadLatestScanAsync(blobServiceClient.AccountName, container.Name).ConfigureAwait(false); containerScanInfo = new ContainerScanInfo() { Registrations = new List <ITriggerExecutor <BlobTriggerExecutorContext> >(), LastSweepCycleLatestModified = latestStoredScan ?? DateTime.MinValue, CurrentSweepCycleLatestModified = DateTime.MinValue, ContinuationToken = null }; Logger.InitializedScanInfo(_logger, container.Name, containerScanInfo.LastSweepCycleLatestModified); _scanInfo.Add(container, containerScanInfo); } containerScanInfo.Registrations.Add(triggerExecutor); }
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor, CancellationToken cancellationToken) { // Register and Execute are not concurrency-safe. // Avoiding calling Register while Execute is running is the caller's responsibility. ThrowIfDisposed(); // Register all in logPolling, there is no problem if we get 2 notifications of the new blob await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken); ContainerScanInfo containerScanInfo; if (!_scanInfo.TryGetValue(container, out containerScanInfo)) { containerScanInfo = new ContainerScanInfo() { Registrations = new List <ITriggerExecutor <IStorageBlob> >(), LastSweepCycleLatestModified = DateTime.MinValue, CurrentSweepCycleLatestModified = DateTime.MinValue, ContinuationToken = null }; _scanInfo.Add(container, containerScanInfo); } containerScanInfo.Registrations.Add(triggerExecutor); }
/// <summary> /// This method is called each polling interval for all containers. The method divides the /// budget of allocated number of blobs to query, for each container we query a page of /// that size and we keep the continuation token for the next time. AS a curser, we use /// the time stamp when the current cycle on the container started. blobs newer than that /// time will be considered new and registrations will be notified /// </summary> /// <param name="container"></param> /// <param name="containerScanInfo"> Information that includes the last cycle start /// the continuation token and the current cycle start for a container</param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <IEnumerable <IStorageBlob> > PollNewBlobsAsync( IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken) { IEnumerable <IStorageListBlobItem> currentBlobs; IStorageBlobResultSegment blobSegment; int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count; BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken; // if starting the cycle, keep the current time stamp to be used as curser if (continuationToken == null) { containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow; } try { blobSegment = await container.ListBlobsSegmentedAsync(prefix : null, useFlatBlobListing : true, blobListingDetails : BlobListingDetails.None, maxResults : blobPollLimitPerContainer, currentToken : continuationToken, options : null, operationContext : null, cancellationToken : cancellationToken); currentBlobs = blobSegment.Results; } catch (StorageException exception) { if (exception.IsNotFound()) { return(Enumerable.Empty <IStorageBlob>()); } else { throw; } } List <IStorageBlob> newBlobs = new List <IStorageBlob>(); // Type cast to IStorageBlob is safe due to useFlatBlobListing: true above. foreach (IStorageBlob currentBlob in currentBlobs) { cancellationToken.ThrowIfCancellationRequested(); IStorageBlobProperties properties = currentBlob.Properties; DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime) { newBlobs.Add(currentBlob); } } // record continuation token for next chunk retrieval containerScanInfo.ContinuationToken = blobSegment.ContinuationToken; // if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime if (blobSegment.ContinuationToken == null) { containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime; } return(newBlobs); }
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken, List <IStorageBlob> failedNotifications) { cancellationToken.ThrowIfCancellationRequested(); DateTime lastScan = containerScanInfo.LastSweepCycleLatestModified; IEnumerable <IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerScanInfo, cancellationToken); foreach (IStorageBlob newBlob in newBlobs) { cancellationToken.ThrowIfCancellationRequested(); await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken); } // if the 'LatestModified' has changed, update it in the manager if (containerScanInfo.LastSweepCycleLatestModified > lastScan) { DateTime latestScan = containerScanInfo.LastSweepCycleLatestModified; // It's possible that we had some blobs that we failed to move to the queue. We want to make sure // we continue to find these if the host needs to restart. if (failedNotifications.Any()) { latestScan = failedNotifications.Min(n => n.Properties.LastModified.Value.UtcDateTime); } // Store our timestamp slightly earlier than the last timestamp. This is a failsafe for any blobs that created // milliseconds after our last scan (blob timestamps round to the second). This way we make sure to pick those // up on a host restart. await _blobScanInfoManager.UpdateLatestScanAsync(container.ServiceClient.Credentials.AccountName, container.Name, latestScan.AddMilliseconds(-1)); } }
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerInfo, CancellationToken cancellationToken, List <IStorageBlob> failedNotifications) { cancellationToken.ThrowIfCancellationRequested(); IEnumerable <IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerInfo, cancellationToken); foreach (IStorageBlob newBlob in newBlobs) { cancellationToken.ThrowIfCancellationRequested(); await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken); } }
private async Task PollAndNotify(BlobContainerClient container, ContainerScanInfo containerScanInfo, List <BlobNotification> failedNotifications, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); DateTime lastScan = containerScanInfo.LastSweepCycleLatestModified; // For tracking string clientRequestId = Guid.NewGuid().ToString(); IEnumerable <BlobBaseClient> newBlobs = await PollNewBlobsAsync(container, containerScanInfo, clientRequestId, cancellationToken).ConfigureAwait(false); foreach (var newBlob in newBlobs) { cancellationToken.ThrowIfCancellationRequested(); await NotifyRegistrationsAsync(new BlobWithContainer <BlobBaseClient>(container, newBlob), failedNotifications, clientRequestId, cancellationToken).ConfigureAwait(false); } // if the 'LatestModified' has changed, update it in the manager if (containerScanInfo.LastSweepCycleLatestModified > lastScan) { DateTime latestScan = containerScanInfo.LastSweepCycleLatestModified; // It's possible that we had some blobs that we failed to move to the queue. We want to make sure // we continue to find these if the host needs to restart. if (failedNotifications.Any()) { // TODO (kasobol-msft) this call to GetProperties is suboptimal figure out how to propagate data from listing here. latestScan = failedNotifications.Select(p => p.Blob).Min(n => n.BlobClient.GetProperties().Value.LastModified.UtcDateTime); } // Store our timestamp slightly earlier than the last timestamp. This is a failsafe for any blobs that created // milliseconds after our last scan (blob timestamps round to the second). This way we make sure to pick those // up on a host restart. await _blobScanInfoManager.UpdateLatestScanAsync(container.AccountName, container.Name, latestScan.AddMilliseconds(-1)).ConfigureAwait(false); } }
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor, CancellationToken cancellationToken) { // Register and Execute are not concurrency-safe. // Avoiding calling Register while Execute is running is the caller's responsibility. ThrowIfDisposed(); // Register all in logPolling, there is no problem if we get 2 notifications of the new blob await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken); ContainerScanInfo containerScanInfo; if (!_scanInfo.TryGetValue(container, out containerScanInfo)) { containerScanInfo = new ContainerScanInfo() { Registrations = new List<ITriggerExecutor<IStorageBlob>>(), LastSweepCycleStartTime = DateTime.MinValue, CurrentSweepCycleStartTime = DateTime.MinValue, ContinuationToken = null }; _scanInfo.Add(container, containerScanInfo); } containerScanInfo.Registrations.Add(triggerExecutor); }
/// <summary> /// This method is called each polling interval for all containers. The method divides the /// budget of allocated number of blobs to query, for each container we query a page of /// that size and we keep the continuation token for the next time. AS a curser, we use /// the time stamp when the current cycle on the container started. blobs newer than that /// time will be considered new and registrations will be notified /// </summary> /// <param name="container"></param> /// <param name="containerScanInfo"> Information that includes the last cycle start /// the continuation token and the current cycle start for a container</param> /// <param name="clientRequestId"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <IEnumerable <BlobBaseClient> > PollNewBlobsAsync( BlobContainerClient container, ContainerScanInfo containerScanInfo, string clientRequestId, CancellationToken cancellationToken) { IEnumerable <BlobItem> currentBlobs; int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count; string continuationToken = containerScanInfo.ContinuationToken; Page <BlobItem> page; // if starting the cycle, reset the sweep time if (continuationToken == null) { containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue; } Stopwatch sw = Stopwatch.StartNew(); try { AsyncPageable <BlobItem> blobsAsyncPageable = container.GetBlobsAsync(cancellationToken: cancellationToken); IAsyncEnumerable <Page <BlobItem> > pages = blobsAsyncPageable.AsPages(continuationToken: continuationToken, pageSizeHint: blobPollLimitPerContainer); IAsyncEnumerator <Page <BlobItem> > pagesEnumerator = pages.GetAsyncEnumerator(cancellationToken); if (await pagesEnumerator.MoveNextAsync().ConfigureAwait(false)) { page = pagesEnumerator.Current; currentBlobs = page.Values; } else { return(Enumerable.Empty <BlobBaseClient>()); } } catch (RequestFailedException exception) { if (exception.IsNotFound()) { Logger.ContainerDoesNotExist(_logger, container.Name); return(Enumerable.Empty <BlobBaseClient>()); } else { throw; } } List <BlobBaseClient> newBlobs = new List <BlobBaseClient>(); // Type cast to IStorageBlob is safe due to useFlatBlobListing: true above. foreach (BlobItem currentBlob in currentBlobs) { cancellationToken.ThrowIfCancellationRequested(); var properties = currentBlob.Properties; DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified) { containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp; } // Blob timestamps are rounded to the nearest second, so make sure we continue to check // the previous timestamp to catch any blobs that came in slightly after our previous poll. if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified) { newBlobs.Add(container.GetBlobClient(currentBlob.Name)); } } Logger.PollBlobContainer(_logger, container.Name, containerScanInfo.LastSweepCycleLatestModified, clientRequestId, newBlobs.Count, sw.ElapsedMilliseconds, !string.IsNullOrWhiteSpace(page.ContinuationToken)); // record continuation token for next chunk retrieval containerScanInfo.ContinuationToken = page.ContinuationToken; // if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime, if changed if (page.ContinuationToken == null && containerScanInfo.CurrentSweepCycleLatestModified > containerScanInfo.LastSweepCycleLatestModified) { containerScanInfo.LastSweepCycleLatestModified = containerScanInfo.CurrentSweepCycleLatestModified; } return(newBlobs); }
/// <summary> /// This method is called each polling interval for all containers. The method divides the /// budget of allocated number of blobs to query, for each container we query a page of /// that size and we keep the continuation token for the next time. AS a curser, we use /// the time stamp when the current cycle on the container started. blobs newer than that /// time will be considered new and registrations will be notified /// </summary> /// <param name="container"></param> /// <param name="containerScanInfo"> Information that includes the last cycle start /// the continuation token and the current cycle start for a container</param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task<IEnumerable<IStorageBlob>> PollNewBlobsAsync( IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken) { IEnumerable<IStorageListBlobItem> currentBlobs; IStorageBlobResultSegment blobSegment; int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count; BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken; // if starting the cycle, keep the current time stamp to be used as curser if (continuationToken == null) { containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow; } try { blobSegment = await container.ListBlobsSegmentedAsync(prefix: null, useFlatBlobListing: true, blobListingDetails: BlobListingDetails.None, maxResults: blobPollLimitPerContainer, currentToken: continuationToken, options: null, operationContext: null, cancellationToken: cancellationToken); currentBlobs = blobSegment.Results; } catch (StorageException exception) { if (exception.IsNotFound()) { return Enumerable.Empty<IStorageBlob>(); } else { throw; } } List<IStorageBlob> newBlobs = new List<IStorageBlob>(); // Type cast to IStorageBlob is safe due to useFlatBlobListing: true above. foreach (IStorageBlob currentBlob in currentBlobs) { cancellationToken.ThrowIfCancellationRequested(); IStorageBlobProperties properties = currentBlob.Properties; DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime) { newBlobs.Add(currentBlob); } } // record continuation token for next chunk retrieval containerScanInfo.ContinuationToken = blobSegment.ContinuationToken; // if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime if (blobSegment.ContinuationToken == null) { containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime; } return newBlobs; }
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerInfo, CancellationToken cancellationToken, List<IStorageBlob> failedNotifications) { cancellationToken.ThrowIfCancellationRequested(); IEnumerable<IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerInfo, cancellationToken); foreach (IStorageBlob newBlob in newBlobs) { cancellationToken.ThrowIfCancellationRequested(); await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken); } }
/// <summary> /// This method is called each polling interval for all containers. The method divides the /// budget of allocated number of blobs to query, for each container we query a page of /// that size and we keep the continuation token for the next time. AS a curser, we use /// the time stamp when the current cycle on the container started. blobs newer than that /// time will be considered new and registrations will be notified /// </summary> /// <param name="container"></param> /// <param name="containerScanInfo"> Information that includes the last cycle start /// the continuation token and the current cycle start for a container</param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <IEnumerable <ICloudBlob> > PollNewBlobsAsync( CloudBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken) { IEnumerable <IListBlobItem> currentBlobs; BlobResultSegment blobSegment; int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count; BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken; // if starting the cycle, reset the sweep time if (continuationToken == null) { containerScanInfo.CurrentSweepCycleLatestModified = DateTime.MinValue; } try { blobSegment = await container.ListBlobsSegmentedAsync(prefix : null, useFlatBlobListing : true, blobListingDetails : BlobListingDetails.None, maxResults : blobPollLimitPerContainer, currentToken : continuationToken, options : null, operationContext : null, cancellationToken : cancellationToken); currentBlobs = blobSegment.Results; } catch (StorageException exception) { if (exception.IsNotFound()) { return(Enumerable.Empty <ICloudBlob>()); } else { throw; } } List <ICloudBlob> newBlobs = new List <ICloudBlob>(); // Type cast to IStorageBlob is safe due to useFlatBlobListing: true above. foreach (ICloudBlob currentBlob in currentBlobs) { cancellationToken.ThrowIfCancellationRequested(); var properties = currentBlob.Properties; DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime; if (lastModifiedTimestamp > containerScanInfo.CurrentSweepCycleLatestModified) { containerScanInfo.CurrentSweepCycleLatestModified = lastModifiedTimestamp; } // Blob timestamps are rounded to the nearest second, so make sure we continue to check // the previous timestamp to catch any blobs that came in slightly after our previous poll. if (lastModifiedTimestamp >= containerScanInfo.LastSweepCycleLatestModified) { newBlobs.Add(currentBlob); } } // record continuation token for next chunk retrieval containerScanInfo.ContinuationToken = blobSegment.ContinuationToken; // if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime, if changed if (blobSegment.ContinuationToken == null && containerScanInfo.CurrentSweepCycleLatestModified > containerScanInfo.LastSweepCycleLatestModified) { containerScanInfo.LastSweepCycleLatestModified = containerScanInfo.CurrentSweepCycleLatestModified; } return(newBlobs); }