public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor, CancellationToken cancellationToken) { ThrowIfDisposed(); // Initial background scans for all containers happen on first Execute call. // Prevent accidental late registrations. // (Also prevents incorrect concurrent execution of Register with Execute.) if (_initialScanThread.ThreadState != ThreadState.Unstarted) { throw new InvalidOperationException("All registrations must be created before execution begins."); } ICollection<ITriggerExecutor<IStorageBlob>> containerRegistrations; if (_registrations.ContainsKey(container)) { containerRegistrations = _registrations[container]; } else { containerRegistrations = new List<ITriggerExecutor<IStorageBlob>>(); _registrations.Add(container, containerRegistrations); } containerRegistrations.Add(triggerExecutor); IStorageBlobClient client = container.ServiceClient; if (!_logListeners.ContainsKey(client)) { BlobLogListener logListener = await BlobLogListener.CreateAsync(client, cancellationToken); _logListeners.Add(client, logListener); } }
public Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor, CancellationToken cancellationToken) { // Register and Execute are not concurrency-safe. // Avoiding calling Register while Execute is running is the caller's responsibility. ICollection<ITriggerExecutor<IStorageBlob>> containerRegistrations; if (_registrations.ContainsKey(container)) { containerRegistrations = _registrations[container]; } else { containerRegistrations = new List<ITriggerExecutor<IStorageBlob>>(); _registrations.Add(container, containerRegistrations); } containerRegistrations.Add(triggerExecutor); if (!_lastModifiedTimestamps.ContainsKey(container)) { _lastModifiedTimestamps.Add(container, DateTime.MinValue); } return Task.FromResult(0); }
public static void Start(this IBlobListenerStrategy strategy, IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor) { if (strategy == null) { throw new ArgumentNullException("strategy"); } strategy.Start(); }
public static void Register(this IBlobListenerStrategy strategy, IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor) { if (strategy == null) { throw new ArgumentNullException("strategy"); } strategy.RegisterAsync(container, triggerExecutor, CancellationToken.None).GetAwaiter().GetResult(); }
public Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor, CancellationToken cancellationToken) { if (_started) { throw new InvalidOperationException( "Registrations may not be added while the shared listener is running."); } return _strategy.RegisterAsync(container, triggerExecutor, cancellationToken); }
public QueueListener(IStorageQueue queue, IStorageQueue poisonQueue, ITriggerExecutor<IStorageQueueMessage> triggerExecutor, IDelayStrategy delayStrategy, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, TraceWriter trace, SharedQueueWatcher sharedWatcher, IQueueConfiguration queueConfiguration) { if (trace == null) { throw new ArgumentNullException("trace"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (queueConfiguration.BatchSize <= 0) { throw new ArgumentException("BatchSize must be greater than zero."); } if (queueConfiguration.MaxDequeueCount <= 0) { throw new ArgumentException("MaxDequeueCount must be greater than zero."); } _timer = new TaskSeriesTimer(this, backgroundExceptionDispatcher, Task.Delay(0)); _queue = queue; _poisonQueue = poisonQueue; _triggerExecutor = triggerExecutor; _delayStrategy = delayStrategy; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _trace = trace; _queueConfiguration = queueConfiguration; if (sharedWatcher != null) { // Call Notify whenever a function adds a message to this queue. sharedWatcher.Register(queue.Name, this); _sharedWatcher = sharedWatcher; } EventHandler poisonMessageEventHandler = _sharedWatcher != null ? OnMessageAddedToPoisonQueue : (EventHandler)null; _queueProcessor = CreateQueueProcessor( _queue.SdkObject, _poisonQueue != null ? _poisonQueue.SdkObject : null, _trace, _queueConfiguration, poisonMessageEventHandler); }
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor, CancellationToken cancellationToken) { // Register and Execute are not concurrency-safe. // Avoiding calling Register while Execute is running is the caller's responsibility. ThrowIfDisposed(); // Register all in logPolling, there is no problem if we get 2 notifications of the new blob await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken); ContainerScanInfo containerScanInfo; if (!_scanInfo.TryGetValue(container, out containerScanInfo)) { containerScanInfo = new ContainerScanInfo() { Registrations = new List<ITriggerExecutor<IStorageBlob>>(), LastSweepCycleStartTime = DateTime.MinValue, CurrentSweepCycleStartTime = DateTime.MinValue, ContinuationToken = null }; _scanInfo.Add(container, containerScanInfo); } containerScanInfo.Registrations.Add(triggerExecutor); }
public async Task RegisterAsync(BlobServiceClient blobServiceClient, BlobContainerClient container, ITriggerExecutor <BlobTriggerExecutorContext> triggerExecutor, CancellationToken cancellationToken) { // Register and Execute are not concurrency-safe. // Avoiding calling Register while Execute is running is the caller's responsibility. ThrowIfDisposed(); // Register all in logPolling, there is no problem if we get 2 notifications of the new blob await _pollLogStrategy.RegisterAsync(blobServiceClient, container, triggerExecutor, cancellationToken).ConfigureAwait(false); if (!_scanInfo.TryGetValue(container, out ContainerScanInfo containerScanInfo)) { // First, try to load serialized scanInfo for this container. DateTime?latestStoredScan = await _blobScanInfoManager.LoadLatestScanAsync(blobServiceClient.AccountName, container.Name).ConfigureAwait(false); containerScanInfo = new ContainerScanInfo() { Registrations = new List <ITriggerExecutor <BlobTriggerExecutorContext> >(), LastSweepCycleLatestModified = latestStoredScan ?? DateTime.MinValue, CurrentSweepCycleLatestModified = DateTime.MinValue, ContinuationToken = null }; Logger.InitializedScanInfo(_logger, container.Name, containerScanInfo.LastSweepCycleLatestModified); _scanInfo.Add(container, containerScanInfo); } containerScanInfo.Registrations.Add(triggerExecutor); }
public QueueListener(CloudQueue queue, CloudQueue poisonQueue, ITriggerExecutor <CloudQueueMessage> triggerExecutor, IWebJobsExceptionHandler exceptionHandler, ILoggerFactory loggerFactory, SharedQueueWatcher sharedWatcher, QueuesOptions queueOptions, IQueueProcessorFactory queueProcessorFactory, TimeSpan?maxPollingInterval = null) { if (queueOptions == null) { throw new ArgumentNullException(nameof(queueOptions)); } if (queueProcessorFactory == null) { throw new ArgumentNullException(nameof(queueProcessorFactory)); } if (queueOptions.BatchSize <= 0) { throw new ArgumentException("BatchSize must be greater than zero."); } if (queueOptions.MaxDequeueCount <= 0) { throw new ArgumentException("MaxDequeueCount must be greater than zero."); } _timer = new TaskSeriesTimer(this, exceptionHandler, Task.Delay(0)); _queue = queue; _poisonQueue = poisonQueue; _triggerExecutor = triggerExecutor; _exceptionHandler = exceptionHandler; _queueOptions = queueOptions; // if the function runs longer than this, the invisibility will be updated // on a timer periodically for the duration of the function execution _visibilityTimeout = TimeSpan.FromMinutes(10); if (sharedWatcher != null) { // Call Notify whenever a function adds a message to this queue. sharedWatcher.Register(queue.Name, this); _sharedWatcher = sharedWatcher; } EventHandler <PoisonMessageEventArgs> poisonMessageEventHandler = _sharedWatcher != null ? OnMessageAddedToPoisonQueue : (EventHandler <PoisonMessageEventArgs>)null; _queueProcessor = CreateQueueProcessor(_queue, _poisonQueue, loggerFactory, queueProcessorFactory, _queueOptions, poisonMessageEventHandler); TimeSpan maximumInterval = _queueProcessor.MaxPollingInterval; if (maxPollingInterval.HasValue && maximumInterval > maxPollingInterval.Value) { // enforce the maximum polling interval if specified maximumInterval = maxPollingInterval.Value; } _delayStrategy = new RandomizedExponentialBackoffStrategy(QueuePollingIntervals.Minimum, maximumInterval); }
public QueueListener(QueueClient queue, QueueClient poisonQueue, ITriggerExecutor <QueueMessage> triggerExecutor, IWebJobsExceptionHandler exceptionHandler, ILoggerFactory loggerFactory, SharedQueueWatcher sharedWatcher, QueuesOptions queueOptions, IQueueProcessorFactory queueProcessorFactory, FunctionDescriptor functionDescriptor, string functionId = null, TimeSpan?maxPollingInterval = null) { if (queueOptions == null) { throw new ArgumentNullException(nameof(queueOptions)); } if (queueProcessorFactory == null) { throw new ArgumentNullException(nameof(queueProcessorFactory)); } if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } if (queueOptions.BatchSize <= 0) { throw new ArgumentException("BatchSize must be greater than zero."); } if (queueOptions.MaxDequeueCount <= 0) { throw new ArgumentException("MaxDequeueCount must be greater than zero."); } _timer = new TaskSeriesTimer(this, exceptionHandler, Task.Delay(0)); _queue = queue; _poisonQueue = poisonQueue; _triggerExecutor = triggerExecutor; _exceptionHandler = exceptionHandler; _queueOptions = queueOptions; _logger = loggerFactory.CreateLogger <QueueListener>(); _functionDescriptor = functionDescriptor ?? throw new ArgumentNullException(nameof(functionDescriptor)); _functionId = functionId ?? _functionDescriptor.Id; // if the function runs longer than this, the invisibility will be updated // on a timer periodically for the duration of the function execution _visibilityTimeout = TimeSpan.FromMinutes(10); if (sharedWatcher != null) { // Call Notify whenever a function adds a message to this queue. sharedWatcher.Register(queue.Name, this); _sharedWatcher = sharedWatcher; } EventHandler <PoisonMessageEventArgs> poisonMessageEventHandler = _sharedWatcher != null ? OnMessageAddedToPoisonQueue : (EventHandler <PoisonMessageEventArgs>)null; _queueProcessor = CreateQueueProcessor(_queue, _poisonQueue, loggerFactory, queueProcessorFactory, _queueOptions, poisonMessageEventHandler); TimeSpan maximumInterval = _queueProcessor.MaxPollingInterval; if (maxPollingInterval.HasValue && maximumInterval > maxPollingInterval.Value) { // enforce the maximum polling interval if specified maximumInterval = maxPollingInterval.Value; } _delayStrategy = new RandomizedExponentialBackoffStrategy(SharedQueuePollingIntervals.Minimum, maximumInterval); _scaleMonitorDescriptor = new ScaleMonitorDescriptor($"{_functionId}-QueueTrigger-{_queue.Name}".ToLower(CultureInfo.InvariantCulture)); _shutdownCancellationTokenSource = new CancellationTokenSource(); }
public Task RegisterAsync(BlobServiceClient blobServiceClient, BlobContainerClient container, ITriggerExecutor <BlobTriggerExecutorContext> triggerExecutor, CancellationToken cancellationToken) { if (_started) { throw new InvalidOperationException( "Registrations may not be added while the shared listener is running."); } return(_strategy.RegisterAsync(blobServiceClient, container, triggerExecutor, cancellationToken)); }
public async Task RegisterAsync(BlobServiceClient blobServiceClient, BlobContainerClient container, ITriggerExecutor <BlobTriggerExecutorContext> triggerExecutor, CancellationToken cancellationToken) { ThrowIfDisposed(); // Initial background scans for all containers happen on first Execute call. // Prevent accidental late registrations. // (Also prevents incorrect concurrent execution of Register with Execute.) if (_initialScanThread.ThreadState != ThreadState.Unstarted) { throw new InvalidOperationException("All registrations must be created before execution begins."); } ICollection <ITriggerExecutor <BlobTriggerExecutorContext> > containerRegistrations; if (_registrations.ContainsKey(container)) { containerRegistrations = _registrations[container]; } else { containerRegistrations = new List <ITriggerExecutor <BlobTriggerExecutorContext> >(); _registrations.Add(container, containerRegistrations); } containerRegistrations.Add(triggerExecutor); if (!_logListeners.ContainsKey(blobServiceClient)) { BlobLogListener logListener = await BlobLogListener.CreateAsync(blobServiceClient, _exceptionHandler, _logger, cancellationToken).ConfigureAwait(false); _logListeners.Add(blobServiceClient, logListener); } }
public void ExecuteAsync_IfLeasedIncompleteReceipt_EnqueuesMessageMarksCompletedReleasesLeaseAndReturnsSuccessResult() { // Arrange string expectedFunctionId = "FunctionId"; BlobTriggerExecutorContext context = CreateExecutorContext(); string expectedETag = context.Blob.BlobClient.GetProperties().Value.ETag.ToString(); IBlobPathSource input = CreateBlobPath(context.Blob); Mock <IBlobReceiptManager> managerMock = CreateReceiptManagerReferenceMock(); managerMock .Setup(m => m.TryReadAsync(It.IsAny <BlockBlobClient>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(BlobReceipt.Incomplete)); managerMock .Setup(m => m.TryAcquireLeaseAsync(It.IsAny <BlockBlobClient>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult("LeaseId")); managerMock .Setup(m => m.MarkCompletedAsync(It.IsAny <BlockBlobClient>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(0)) .Verifiable(); managerMock .Setup(m => m.ReleaseLeaseAsync(It.IsAny <BlockBlobClient>(), It.IsAny <string>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(0)) .Verifiable(); IBlobReceiptManager receiptManager = managerMock.Object; Mock <IBlobTriggerQueueWriter> queueWriterMock = new Mock <IBlobTriggerQueueWriter>(MockBehavior.Strict); queueWriterMock .Setup(w => w.EnqueueAsync(It.IsAny <BlobTriggerMessage>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(("testQueueName", "testMessageId"))); IBlobTriggerQueueWriter queueWriter = queueWriterMock.Object; ITriggerExecutor <BlobTriggerExecutorContext> product = CreateProductUnderTest(expectedFunctionId, input, receiptManager, queueWriter); // Act Task <FunctionResult> task = product.ExecuteAsync(context, CancellationToken.None); // Assert task.WaitUntilCompleted(); queueWriterMock .Verify( w => w.EnqueueAsync(It.Is <BlobTriggerMessage>(m => m != null && m.FunctionId == expectedFunctionId /*&& m.BlobType == StorageBlobType.BlockBlob $$$ */ && m.BlobName == context.Blob.BlobClient.Name && m.ContainerName == context.Blob.BlobClient.BlobContainerName && m.ETag == expectedETag), It.IsAny <CancellationToken>()), Times.Once()); managerMock.Verify(); Assert.True(task.Result.Succeeded); // Validate log is written var logMessage = _loggerProvider.GetAllLogMessages().Single(); Assert.AreEqual("BlobMessageEnqueued", logMessage.EventId.Name); Assert.AreEqual(LogLevel.Debug, logMessage.Level); Assert.AreEqual(7, logMessage.State.Count()); Assert.AreEqual("FunctionIdLogName", logMessage.GetStateValue <string>("functionName")); Assert.AreEqual(context.Blob.BlobClient.Name, logMessage.GetStateValue <string>("blobName")); Assert.AreEqual("testQueueName", logMessage.GetStateValue <string>("queueName")); Assert.AreEqual("testMessageId", logMessage.GetStateValue <string>("messageId")); Assert.AreEqual(context.PollId, logMessage.GetStateValue <string>("pollId")); Assert.AreEqual(context.TriggerSource, logMessage.GetStateValue <BlobTriggerSource>("triggerSource")); Assert.True(!string.IsNullOrWhiteSpace(logMessage.GetStateValue <string>("{OriginalFormat}"))); }
public Task RegisterAsync(BlobServiceClient blobServiceClient, BlobContainerClient container, ITriggerExecutor <BlobTriggerExecutorContext> triggerExecutor, CancellationToken cancellationToken) { // Register and Execute are not concurrency-safe. // Avoiding calling Register while Execute is running is the caller's responsibility. ICollection <ITriggerExecutor <BlobTriggerExecutorContext> > containerRegistrations; if (_registrations.ContainsKey(container)) { containerRegistrations = _registrations[container]; } else { containerRegistrations = new List <ITriggerExecutor <BlobTriggerExecutorContext> >(); _registrations.Add(container, containerRegistrations); } containerRegistrations.Add(triggerExecutor); if (!_lastModifiedTimestamps.ContainsKey(container)) { _lastModifiedTimestamps.Add(container, DateTime.MinValue); } return(Task.FromResult(0)); }