public QueueTriggerBinding(string parameterName, CloudQueue queue, ITriggerDataArgumentBinding <CloudQueueMessage> argumentBinding, QueuesOptions queueOptions, IWebJobsExceptionHandler exceptionHandler, SharedQueueWatcher messageEnqueuedWatcherSetter, ILoggerFactory loggerFactory, IQueueProcessorFactory queueProcessorFactory) { _queue = queue ?? throw new ArgumentNullException(nameof(queue)); _argumentBinding = argumentBinding ?? throw new ArgumentNullException(nameof(argumentBinding)); _bindingDataContract = CreateBindingDataContract(argumentBinding.BindingDataContract); _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter)); _parameterName = parameterName; _loggerFactory = loggerFactory; _queueProcessorFactory = queueProcessorFactory; _converter = CreateConverter(queue); }
public SharedBlobQueueListenerFactory( QueueServiceClient hostQueueServiceClient, SharedQueueWatcher sharedQueueWatcher, QueueClient hostBlobTriggerQueue, BlobsOptions blobsOptions, IWebJobsExceptionHandler exceptionHandler, ILoggerFactory loggerFactory, IBlobWrittenWatcher blobWrittenWatcher, FunctionDescriptor functionDescriptor, BlobTriggerSource blobTriggerSource) { _hostQueueServiceClient = hostQueueServiceClient ?? throw new ArgumentNullException(nameof(hostQueueServiceClient)); _sharedQueueWatcher = sharedQueueWatcher ?? throw new ArgumentNullException(nameof(sharedQueueWatcher)); _hostBlobTriggerQueue = hostBlobTriggerQueue ?? throw new ArgumentNullException(nameof(hostBlobTriggerQueue)); _blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _loggerFactory = loggerFactory; _blobWrittenWatcher = blobWrittenWatcher; _functionDescriptor = functionDescriptor; _blobTriggerSource = blobTriggerSource; }
public QueueListenerFactory( QueueServiceClient queueServiceClient, QueueClient queue, QueuesOptions queueOptions, IWebJobsExceptionHandler exceptionHandler, SharedQueueWatcher messageEnqueuedWatcherSetter, ILoggerFactory loggerFactory, ITriggeredFunctionExecutor executor, IQueueProcessorFactory queueProcessorFactory, FunctionDescriptor descriptor) { _queue = queue ?? throw new ArgumentNullException(nameof(queue)); _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter)); _executor = executor ?? throw new ArgumentNullException(nameof(executor)); _descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor)); _poisonQueue = CreatePoisonQueueReference(queueServiceClient, queue.Name); _loggerFactory = loggerFactory; _queueProcessorFactory = queueProcessorFactory; }
// initialize following fields async // _triggerExecutor --> register messageHandler // _sharedQueuelistener --> dequeue messages and call messageHandler // _sharedQueueWriter --> enqueue messages internal async Task InitializeAsync(CancellationToken cancellationToken) { if (_state != State.Created) { // only initialized once, since _state is incremental throw new InvalidOperationException(ErrorMessage(State.Created, _state)); } // concurrent dictionary that we can register messageHandler _triggerExecutor = new SharedQueueExecutor(); try { IStorageQueueClient primaryQueueClient = (await _accountProvider.GetStorageAccountAsync(cancellationToken)).CreateQueueClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken); // one host level shared queue // queue is not created here, only after 1st message added IStorageQueue sharedQueue = primaryQueueClient.GetQueueReference(HostQueueNames.GetHostSharedQueueName(hostId)); // default host level poison queue IStorageQueue sharedPoisonQueue = primaryQueueClient.GetQueueReference(HostQueueNames.GetHostSharedPoisonQueueName(hostId)); // queueWatcher will update queueListener's polling interval when queueWriter performes an enqueue operation SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); _sharedQueueWriter = new SharedQueueWriter(sharedQueue, sharedQueueWatcher); // use default poisonQueue setup _sharedQueuelistener = new QueueListener(sharedQueue, sharedPoisonQueue, _triggerExecutor, _exceptionHandler, _loggerFactory, sharedQueueWatcher, _queueConfiguration); } catch (Exception ex) { // initialization exception will fail all registrations _initializationEx = ex; } _state = State.Initialized; }
private IListener CreateQueueMessageToTriggerExecutionListener( ISharedContextProvider sharedContextProvider, SharedQueueWatcher sharedQueueWatcher, IStorageQueueClient queueClient, IStorageQueue hostBlobTriggerQueue, IStorageBlobClient blobClient, IBlobWrittenWatcher blobWrittenWatcher) { SharedBlobQueueListener sharedListener = sharedContextProvider.GetOrCreate <SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue, _queueConfiguration, _backgroundExceptionDispatcher, _trace, blobWrittenWatcher)); BlobQueueRegistration registration = new BlobQueueRegistration { Executor = _executor, BlobClient = blobClient }; sharedListener.Register(_functionId, registration); return(new BlobListener(sharedListener)); }
public BlobTriggerBinding(ParameterInfo parameter, BlobServiceClient hostBlobServiceClient, QueueServiceClient hostQueueServiceClient, BlobServiceClient dataBlobServiceClient, QueueServiceClient dataQueueServiceClient, IBlobPathSource path, IHostIdProvider hostIdProvider, QueuesOptions queueOptions, BlobsOptions blobsOptions, IWebJobsExceptionHandler exceptionHandler, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, SharedQueueWatcher messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, IHostSingletonManager singletonManager, ILoggerFactory loggerFactory) { _parameter = parameter ?? throw new ArgumentNullException(nameof(parameter)); _hostBlobServiceClient = hostBlobServiceClient ?? throw new ArgumentNullException(nameof(hostBlobServiceClient)); _hostQueueServiceClient = hostQueueServiceClient ?? throw new ArgumentNullException(nameof(hostQueueServiceClient)); _dataBlobServiceClient = dataBlobServiceClient ?? throw new ArgumentNullException(nameof(dataBlobServiceClient)); _dataQueueServiceClient = dataQueueServiceClient ?? throw new ArgumentNullException(nameof(dataQueueServiceClient)); _accountName = _dataBlobServiceClient.AccountName; _path = path ?? throw new ArgumentNullException(nameof(path)); _hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); _blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter)); _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter)); _sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider)); _singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager)); _loggerFactory = loggerFactory; _converter = CreateConverter(_dataBlobServiceClient); _bindingDataContract = CreateBindingDataContract(path); }
public BlobListenerFactory(IHostIdProvider hostIdProvider, QueuesOptions queueOptions, BlobsOptions blobsOptions, IWebJobsExceptionHandler exceptionHandler, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, SharedQueueWatcher messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, ILoggerFactory loggerFactory, FunctionDescriptor functionDescriptor, BlobServiceClient hostBlobServiceClient, QueueServiceClient hostQueueServiceClient, BlobServiceClient dataBlobServiceClient, QueueServiceClient dataQueueServiceClient, BlobContainerClient container, IBlobPathSource input, ITriggeredFunctionExecutor executor, IHostSingletonManager singletonManager) { _hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); _blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter)); _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter)); _sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider)); _functionDescriptor = functionDescriptor ?? throw new ArgumentNullException(nameof(functionDescriptor)); _loggerFactory = loggerFactory; _hostBlobServiceClient = hostBlobServiceClient ?? throw new ArgumentNullException(nameof(hostBlobServiceClient)); _hostQueueServiceClient = hostQueueServiceClient ?? throw new ArgumentNullException(nameof(hostQueueServiceClient)); _dataBlobServiceClient = dataBlobServiceClient ?? throw new ArgumentNullException(nameof(dataBlobServiceClient)); _dataQueueServiceClient = dataQueueServiceClient ?? throw new ArgumentNullException(nameof(dataQueueServiceClient)); _container = container ?? throw new ArgumentNullException(nameof(container)); _input = input ?? throw new ArgumentNullException(nameof(input)); _executor = executor ?? throw new ArgumentNullException(nameof(executor)); _singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager)); }
public async Task <IListener> CreateAsync(CancellationToken cancellationToken) { // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. var primaryQueueClient = _hostQueueServiceClient; var primaryBlobClient = _hostBlobServiceClient; // Important: We're using the storage account of the function target here, which is the account that the // function the listener is for is targeting. This is the account that will be used // to read the trigger blob. var targetBlobClient = _dataBlobServiceClient; var targetQueueClient = _dataQueueServiceClient; string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken).ConfigureAwait(false); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); var hostBlobTriggerQueue = primaryQueueClient.GetQueueClient(hostBlobTriggerQueueName); SharedQueueWatcher sharedQueueWatcher = _messageEnqueuedWatcherSetter; SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>( new SharedBlobListenerFactory(hostId, _hostBlobServiceClient, _exceptionHandler, _blobWrittenWatcherSetter, _loggerFactory.CreateLogger <BlobListener>())); // Register the blob container we wish to monitor with the shared blob listener. await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken).ConfigureAwait(false); // Create a "bridge" listener that will monitor the blob // notification queue and dispatch to the target job function. SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(_hostQueueServiceClient, sharedQueueWatcher, hostBlobTriggerQueue, _blobsOptions, _exceptionHandler, _loggerFactory, sharedBlobListener.BlobWritterWatcher, _functionDescriptor)); var queueListener = new BlobListener(sharedBlobQueueListener); // the client to use for the poison queue // by default this should target the same storage account // as the blob container we're monitoring var poisonQueueClient = targetQueueClient; // Register our function with the shared blob queue listener RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient); // check a flag in the shared context to see if we've created the singleton // shared blob listener in this host instance object singletonListenerCreated = false; if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated)) { // Create a singleton shared blob listener, since we only // want a single instance of the blob poll/scan logic to be running // across host instances var singletonBlobListener = _singletonManager.CreateHostSingletonListener( new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId); _sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true); return(new CompositeListener(singletonBlobListener, queueListener)); } else { // We've already created the singleton blob listener // so just return our queue listener. Note that while we want the // blob scan to be singleton, the shared queue listener needs to run // on ALL instances so load can be scaled out return(queueListener); } }
public SharedBlobQueueListenerFactory(IFunctionExecutor executor, SharedQueueWatcher sharedQueueWatcher, IStorageQueueClient queueClient, IStorageQueue hostBlobTriggerQueue, IStorageBlobClient blobClient, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, TextWriter log, IBlobWrittenWatcher blobWrittenWatcher) { if (executor == null) { throw new ArgumentNullException("executor"); } if (sharedQueueWatcher == null) { throw new ArgumentNullException("sharedQueueWatcher"); } if (queueClient == null) { throw new ArgumentNullException("queueClient"); } if (hostBlobTriggerQueue == null) { throw new ArgumentNullException("hostBlobTriggerQueue"); } if (blobClient == null) { throw new ArgumentNullException("blobClient"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (log == null) { throw new ArgumentNullException("log"); } if (blobWrittenWatcher == null) { throw new ArgumentNullException("blobWrittenWatcher"); } _executor = executor; _sharedQueueWatcher = sharedQueueWatcher; _queueClient = queueClient; _hostBlobTriggerQueue = hostBlobTriggerQueue; _blobClient = blobClient; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _log = log; _blobWrittenWatcher = blobWrittenWatcher; }
public async Task <IListener> CreateAsync(CancellationToken cancellationToken) { SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>( new SharedBlobListenerFactory(_hostAccount, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter)); // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. IStorageQueueClient queueClient = _hostAccount.CreateQueueClient(); IStorageBlobClient blobClient = _hostAccount.CreateBlobClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); IStorageQueue hostBlobTriggerQueue = queueClient.GetQueueReference(hostBlobTriggerQueueName); // Register the blob container we wish to monitor with the shared blob listener. await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, blobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken); // Create a "bridge" listener that will monitor the blob // notification queue and dispatch to the target job function. SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue, _queueConfiguration, _backgroundExceptionDispatcher, _trace, sharedBlobListener.BlobWritterWatcher)); var queueListener = new BlobListener(sharedBlobQueueListener); // Important: We're using the "data account" here, which is the account that the // function the listener is for is targeting. This is the account that will be used // to read the trigger blob. IStorageBlobClient userBlobClient = _dataAccount.CreateBlobClient(); // Register our function with the shared queue listener RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, userBlobClient); // check a flag in the shared context to see if we've created the singleton // shared blob listener in this host instance object singletonListenerCreated = false; if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated)) { // Create a singleton shared blob listener, since we only // want a single instance of the blob poll/scan logic to be running // across host instances var singletonBlobListener = _singletonManager.CreateHostSingletonListener( new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId); _sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true); return(new CompositeListener(singletonBlobListener, queueListener)); } else { // We've already created the singleton blob listener // so just return our queue listener. Note that while we want the // blob scan to be singleton, the shared queue listener needs to run // on ALL instances so load can be scaled out return(queueListener); } }
public BlobTriggerQueueWriter(QueueClient queueClient, SharedQueueWatcher wsharedQueueWatcher) { QueueClient = queueClient; SharedQueueWatcher = wsharedQueueWatcher; }
public BlobTriggerQueueWriterFactory(IHostIdProvider hostIdProvider, QueueServiceClientProvider queueServiceClientProvider, SharedQueueWatcher sharedQueueWatcher) { _hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); _queueServiceClientProvider = queueServiceClientProvider ?? throw new ArgumentNullException(nameof(queueServiceClientProvider)); _sharedQueueWatcher = sharedQueueWatcher ?? throw new ArgumentNullException(nameof(sharedQueueWatcher)); }