public QueueTriggerBinding(string parameterName,
CloudQueue queue,
ITriggerDataArgumentBinding <CloudQueueMessage> argumentBinding,
QueuesOptions queueOptions,
IWebJobsExceptionHandler exceptionHandler,
SharedQueueWatcher messageEnqueuedWatcherSetter,
ILoggerFactory loggerFactory,
IQueueProcessorFactory queueProcessorFactory)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_argumentBinding = argumentBinding ?? throw new ArgumentNullException(nameof(argumentBinding));
_bindingDataContract = CreateBindingDataContract(argumentBinding.BindingDataContract);
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_parameterName = parameterName;
_loggerFactory = loggerFactory;
_queueProcessorFactory = queueProcessorFactory;
_converter = CreateConverter(queue);
}
public SharedBlobQueueListenerFactory(
QueueServiceClient hostQueueServiceClient,
SharedQueueWatcher sharedQueueWatcher,
QueueClient hostBlobTriggerQueue,
BlobsOptions blobsOptions,
IWebJobsExceptionHandler exceptionHandler,
ILoggerFactory loggerFactory,
IBlobWrittenWatcher blobWrittenWatcher,
FunctionDescriptor functionDescriptor,
BlobTriggerSource blobTriggerSource)
{
_hostQueueServiceClient = hostQueueServiceClient ?? throw new ArgumentNullException(nameof(hostQueueServiceClient));
_sharedQueueWatcher = sharedQueueWatcher ?? throw new ArgumentNullException(nameof(sharedQueueWatcher));
_hostBlobTriggerQueue = hostBlobTriggerQueue ?? throw new ArgumentNullException(nameof(hostBlobTriggerQueue));
_blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_loggerFactory = loggerFactory;
_blobWrittenWatcher = blobWrittenWatcher;
_functionDescriptor = functionDescriptor;
_blobTriggerSource = blobTriggerSource;
}
public QueueListenerFactory(
QueueServiceClient queueServiceClient,
QueueClient queue,
QueuesOptions queueOptions,
IWebJobsExceptionHandler exceptionHandler,
SharedQueueWatcher messageEnqueuedWatcherSetter,
ILoggerFactory loggerFactory,
ITriggeredFunctionExecutor executor,
IQueueProcessorFactory queueProcessorFactory,
FunctionDescriptor descriptor)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_executor = executor ?? throw new ArgumentNullException(nameof(executor));
_descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor));
_poisonQueue = CreatePoisonQueueReference(queueServiceClient, queue.Name);
_loggerFactory = loggerFactory;
_queueProcessorFactory = queueProcessorFactory;
}
// initialize following fields async
// _triggerExecutor --> register messageHandler
// _sharedQueuelistener --> dequeue messages and call messageHandler
// _sharedQueueWriter --> enqueue messages
internal async Task InitializeAsync(CancellationToken cancellationToken)
{
if (_state != State.Created)
{
// only initialized once, since _state is incremental
throw new InvalidOperationException(ErrorMessage(State.Created, _state));
}
// concurrent dictionary that we can register messageHandler
_triggerExecutor = new SharedQueueExecutor();
try
{
IStorageQueueClient primaryQueueClient = (await _accountProvider.GetStorageAccountAsync(cancellationToken)).CreateQueueClient();
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
// one host level shared queue
// queue is not created here, only after 1st message added
IStorageQueue sharedQueue = primaryQueueClient.GetQueueReference(HostQueueNames.GetHostSharedQueueName(hostId));
// default host level poison queue
IStorageQueue sharedPoisonQueue = primaryQueueClient.GetQueueReference(HostQueueNames.GetHostSharedPoisonQueueName(hostId));
// queueWatcher will update queueListener's polling interval when queueWriter performes an enqueue operation
SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>(
new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter));
_sharedQueueWriter = new SharedQueueWriter(sharedQueue, sharedQueueWatcher);
// use default poisonQueue setup
_sharedQueuelistener = new QueueListener(sharedQueue, sharedPoisonQueue, _triggerExecutor,
_exceptionHandler, _loggerFactory, sharedQueueWatcher, _queueConfiguration);
}
catch (Exception ex)
{
// initialization exception will fail all registrations
_initializationEx = ex;
}
_state = State.Initialized;
}
private IListener CreateQueueMessageToTriggerExecutionListener(
ISharedContextProvider sharedContextProvider,
SharedQueueWatcher sharedQueueWatcher,
IStorageQueueClient queueClient,
IStorageQueue hostBlobTriggerQueue,
IStorageBlobClient blobClient,
IBlobWrittenWatcher blobWrittenWatcher)
{
SharedBlobQueueListener sharedListener = sharedContextProvider.GetOrCreate <SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue,
_queueConfiguration, _backgroundExceptionDispatcher, _trace, blobWrittenWatcher));
BlobQueueRegistration registration = new BlobQueueRegistration
{
Executor = _executor,
BlobClient = blobClient
};
sharedListener.Register(_functionId, registration);
return(new BlobListener(sharedListener));
}
public BlobTriggerBinding(ParameterInfo parameter,
BlobServiceClient hostBlobServiceClient,
QueueServiceClient hostQueueServiceClient,
BlobServiceClient dataBlobServiceClient,
QueueServiceClient dataQueueServiceClient,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
QueuesOptions queueOptions,
BlobsOptions blobsOptions,
IWebJobsExceptionHandler exceptionHandler,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
SharedQueueWatcher messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
IHostSingletonManager singletonManager,
ILoggerFactory loggerFactory)
{
_parameter = parameter ?? throw new ArgumentNullException(nameof(parameter));
_hostBlobServiceClient = hostBlobServiceClient ?? throw new ArgumentNullException(nameof(hostBlobServiceClient));
_hostQueueServiceClient = hostQueueServiceClient ?? throw new ArgumentNullException(nameof(hostQueueServiceClient));
_dataBlobServiceClient = dataBlobServiceClient ?? throw new ArgumentNullException(nameof(dataBlobServiceClient));
_dataQueueServiceClient = dataQueueServiceClient ?? throw new ArgumentNullException(nameof(dataQueueServiceClient));
_accountName = _dataBlobServiceClient.AccountName;
_path = path ?? throw new ArgumentNullException(nameof(path));
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider));
_singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager));
_loggerFactory = loggerFactory;
_converter = CreateConverter(_dataBlobServiceClient);
_bindingDataContract = CreateBindingDataContract(path);
}
public BlobListenerFactory(IHostIdProvider hostIdProvider,
QueuesOptions queueOptions,
BlobsOptions blobsOptions,
IWebJobsExceptionHandler exceptionHandler,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
SharedQueueWatcher messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
ILoggerFactory loggerFactory,
FunctionDescriptor functionDescriptor,
BlobServiceClient hostBlobServiceClient,
QueueServiceClient hostQueueServiceClient,
BlobServiceClient dataBlobServiceClient,
QueueServiceClient dataQueueServiceClient,
BlobContainerClient container,
IBlobPathSource input,
ITriggeredFunctionExecutor executor,
IHostSingletonManager singletonManager)
{
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions));
_blobsOptions = blobsOptions ?? throw new ArgumentNullException(nameof(blobsOptions));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider));
_functionDescriptor = functionDescriptor ?? throw new ArgumentNullException(nameof(functionDescriptor));
_loggerFactory = loggerFactory;
_hostBlobServiceClient = hostBlobServiceClient ?? throw new ArgumentNullException(nameof(hostBlobServiceClient));
_hostQueueServiceClient = hostQueueServiceClient ?? throw new ArgumentNullException(nameof(hostQueueServiceClient));
_dataBlobServiceClient = dataBlobServiceClient ?? throw new ArgumentNullException(nameof(dataBlobServiceClient));
_dataQueueServiceClient = dataQueueServiceClient ?? throw new ArgumentNullException(nameof(dataQueueServiceClient));
_container = container ?? throw new ArgumentNullException(nameof(container));
_input = input ?? throw new ArgumentNullException(nameof(input));
_executor = executor ?? throw new ArgumentNullException(nameof(executor));
_singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager));
}
public async Task <IListener> CreateAsync(CancellationToken cancellationToken)
{
// Note that these clients are intentionally for the storage account rather than for the dashboard account.
// We use the storage, not dashboard, account for the blob receipt container and blob trigger queues.
var primaryQueueClient = _hostQueueServiceClient;
var primaryBlobClient = _hostBlobServiceClient;
// Important: We're using the storage account of the function target here, which is the account that the
// function the listener is for is targeting. This is the account that will be used
// to read the trigger blob.
var targetBlobClient = _dataBlobServiceClient;
var targetQueueClient = _dataQueueServiceClient;
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken).ConfigureAwait(false);
string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId);
var hostBlobTriggerQueue = primaryQueueClient.GetQueueClient(hostBlobTriggerQueueName);
SharedQueueWatcher sharedQueueWatcher = _messageEnqueuedWatcherSetter;
SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>(
new SharedBlobListenerFactory(hostId, _hostBlobServiceClient, _exceptionHandler, _blobWrittenWatcherSetter, _loggerFactory.CreateLogger <BlobListener>()));
// Register the blob container we wish to monitor with the shared blob listener.
await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient,
hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken).ConfigureAwait(false);
// Create a "bridge" listener that will monitor the blob
// notification queue and dispatch to the target job function.
SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(_hostQueueServiceClient, sharedQueueWatcher, hostBlobTriggerQueue,
_blobsOptions, _exceptionHandler, _loggerFactory, sharedBlobListener.BlobWritterWatcher, _functionDescriptor));
var queueListener = new BlobListener(sharedBlobQueueListener);
// the client to use for the poison queue
// by default this should target the same storage account
// as the blob container we're monitoring
var poisonQueueClient = targetQueueClient;
// Register our function with the shared blob queue listener
RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient);
// check a flag in the shared context to see if we've created the singleton
// shared blob listener in this host instance
object singletonListenerCreated = false;
if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated))
{
// Create a singleton shared blob listener, since we only
// want a single instance of the blob poll/scan logic to be running
// across host instances
var singletonBlobListener = _singletonManager.CreateHostSingletonListener(
new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId);
_sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true);
return(new CompositeListener(singletonBlobListener, queueListener));
}
else
{
// We've already created the singleton blob listener
// so just return our queue listener. Note that while we want the
// blob scan to be singleton, the shared queue listener needs to run
// on ALL instances so load can be scaled out
return(queueListener);
}
}
public SharedBlobQueueListenerFactory(IFunctionExecutor executor,
SharedQueueWatcher sharedQueueWatcher,
IStorageQueueClient queueClient,
IStorageQueue hostBlobTriggerQueue,
IStorageBlobClient blobClient,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
TextWriter log,
IBlobWrittenWatcher blobWrittenWatcher)
{
if (executor == null)
{
throw new ArgumentNullException("executor");
}
if (sharedQueueWatcher == null)
{
throw new ArgumentNullException("sharedQueueWatcher");
}
if (queueClient == null)
{
throw new ArgumentNullException("queueClient");
}
if (hostBlobTriggerQueue == null)
{
throw new ArgumentNullException("hostBlobTriggerQueue");
}
if (blobClient == null)
{
throw new ArgumentNullException("blobClient");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
if (blobWrittenWatcher == null)
{
throw new ArgumentNullException("blobWrittenWatcher");
}
_executor = executor;
_sharedQueueWatcher = sharedQueueWatcher;
_queueClient = queueClient;
_hostBlobTriggerQueue = hostBlobTriggerQueue;
_blobClient = blobClient;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_log = log;
_blobWrittenWatcher = blobWrittenWatcher;
}
public async Task <IListener> CreateAsync(CancellationToken cancellationToken)
{
SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>(
new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter));
SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>(
new SharedBlobListenerFactory(_hostAccount, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter));
// Note that these clients are intentionally for the storage account rather than for the dashboard account.
// We use the storage, not dashboard, account for the blob receipt container and blob trigger queues.
IStorageQueueClient queueClient = _hostAccount.CreateQueueClient();
IStorageBlobClient blobClient = _hostAccount.CreateBlobClient();
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId);
IStorageQueue hostBlobTriggerQueue = queueClient.GetQueueReference(hostBlobTriggerQueueName);
// Register the blob container we wish to monitor with the shared blob listener.
await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, blobClient,
hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken);
// Create a "bridge" listener that will monitor the blob
// notification queue and dispatch to the target job function.
SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue,
_queueConfiguration, _backgroundExceptionDispatcher, _trace, sharedBlobListener.BlobWritterWatcher));
var queueListener = new BlobListener(sharedBlobQueueListener);
// Important: We're using the "data account" here, which is the account that the
// function the listener is for is targeting. This is the account that will be used
// to read the trigger blob.
IStorageBlobClient userBlobClient = _dataAccount.CreateBlobClient();
// Register our function with the shared queue listener
RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, userBlobClient);
// check a flag in the shared context to see if we've created the singleton
// shared blob listener in this host instance
object singletonListenerCreated = false;
if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated))
{
// Create a singleton shared blob listener, since we only
// want a single instance of the blob poll/scan logic to be running
// across host instances
var singletonBlobListener = _singletonManager.CreateHostSingletonListener(
new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId);
_sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true);
return(new CompositeListener(singletonBlobListener, queueListener));
}
else
{
// We've already created the singleton blob listener
// so just return our queue listener. Note that while we want the
// blob scan to be singleton, the shared queue listener needs to run
// on ALL instances so load can be scaled out
return(queueListener);
}
}
public BlobTriggerQueueWriter(QueueClient queueClient, SharedQueueWatcher wsharedQueueWatcher)
{
QueueClient = queueClient;
SharedQueueWatcher = wsharedQueueWatcher;
}
public BlobTriggerQueueWriterFactory(IHostIdProvider hostIdProvider, QueueServiceClientProvider queueServiceClientProvider, SharedQueueWatcher sharedQueueWatcher)
{
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueServiceClientProvider = queueServiceClientProvider ?? throw new ArgumentNullException(nameof(queueServiceClientProvider));
_sharedQueueWatcher = sharedQueueWatcher ?? throw new ArgumentNullException(nameof(sharedQueueWatcher));
}
