public async Task <IListener> CreateAsync(IFunctionExecutor executor, CancellationToken cancellationToken)
{
SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreate <SharedQueueWatcher>(
new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter));
SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreate <SharedBlobListener>(
new SharedBlobListenerFactory(_account, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter));
// Note that these clients are intentionally for the storage account rather than for the dashboard account.
// We use the storage, not dashboard, account for the blob receipt container and blob trigger queues.
IStorageQueueClient queueClient = _account.CreateQueueClient();
IStorageBlobClient blobClient = _account.CreateBlobClient();
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId);
IStorageQueue hostBlobTriggerQueue = queueClient.GetQueueReference(hostBlobTriggerQueueName);
IListener blobDiscoveryToQueueMessageListener = await CreateBlobDiscoveryToQueueMessageListenerAsync(
hostId, sharedBlobListener, blobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken);
IListener queueMessageToTriggerExecutionListener = CreateQueueMessageToTriggerExecutionListener(executor,
_sharedContextProvider, sharedQueueWatcher, queueClient, hostBlobTriggerQueue, blobClient,
sharedBlobListener.BlobWritterWatcher);
IListener compositeListener = new CompositeListener(
blobDiscoveryToQueueMessageListener,
queueMessageToTriggerExecutionListener);
return(compositeListener);
}
private static IStorageQueue CreateQueue(IStorageAccount account, string queueName)
{
IStorageQueueClient client = account.CreateQueueClient();
IStorageQueue queue = client.GetQueueReference(queueName);
queue.CreateIfNotExists();
return queue;
}
public async Task CloudQueueCreate_IfNotExist_CreatesQueue()
{
// Arrange
CloudStorageAccount sdkAccount = CreateSdkAccount();
string queueName = GetQueueName("create-queue");
CloudQueue sdkQueue = CreateSdkQueue(sdkAccount, queueName);
try
{
IStorageAccount product = CreateProductUnderTest(sdkAccount);
IStorageQueueClient client = product.CreateQueueClient();
Assert.NotNull(client); // Guard
IStorageQueue queue = client.GetQueueReference(queueName);
Assert.NotNull(queue); // Guard
// Act
await queue.CreateIfNotExistsAsync(CancellationToken.None);
// Assert
Assert.True(await sdkQueue.ExistsAsync());
}
finally
{
if (await sdkQueue.ExistsAsync())
{
await sdkQueue.DeleteAsync();
}
}
}
public async Task <IBinding> TryCreateAsync(BindingProviderContext context)
{
ParameterInfo parameter = context.Parameter;
QueueAttribute queueAttribute = parameter.GetCustomAttribute <QueueAttribute>(inherit: false);
if (queueAttribute == null)
{
return(null);
}
string queueName = Resolve(queueAttribute.QueueName);
IBindableQueuePath path = BindableQueuePath.Create(queueName);
path.ValidateContractCompatibility(context.BindingDataContract);
IArgumentBinding <IStorageQueue> argumentBinding = _innerProvider.TryCreate(parameter);
if (argumentBinding == null)
{
throw new InvalidOperationException("Can't bind Queue to type '" + parameter.ParameterType + "'.");
}
IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.Parameter, context.CancellationToken);
StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext
{
Parameter = parameter
};
IStorageQueueClient client = account.CreateQueueClient(clientFactoryContext);
IBinding binding = new QueueBinding(parameter.Name, argumentBinding, client, path);
return(binding);
}
public async Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
{
ParameterInfo parameter = context.Parameter;
QueueTriggerAttribute queueTrigger = parameter.GetCustomAttribute <QueueTriggerAttribute>(inherit: false);
if (queueTrigger == null)
{
return(null);
}
string queueName = Resolve(queueTrigger.QueueName);
queueName = NormalizeAndValidate(queueName);
ITriggerDataArgumentBinding <IStorageQueueMessage> argumentBinding = _innerProvider.TryCreate(parameter);
if (argumentBinding == null)
{
throw new InvalidOperationException(
"Can't bind QueueTrigger to type '" + parameter.ParameterType + "'.");
}
IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.CancellationToken);
IStorageQueueClient client = account.CreateQueueClient();
IStorageQueue queue = client.GetQueueReference(queueName);
ITriggerBinding binding = new QueueTriggerBinding(parameter.Name, queue, argumentBinding,
_queueConfiguration, _backgroundExceptionDispatcher, _messageEnqueuedWatcherSetter,
_sharedContextProvider, _log);
return(binding);
}
public void InvokeWithBindingData()
{
// Verify that queue binding pattern has uppercase letters in it. These get normalized to lowercase.
Assert.NotEqual(ProgramWithTriggerAndBindingData.QueueOutName, ProgramWithTriggerAndBindingData.QueueOutName.ToLower());
IStorageAccount account = CreateFakeStorageAccount();
var host = TestHelpers.NewJobHost <ProgramWithTriggerAndBindingData>(account);
var trigger = new ProgramWithTriggerAndBindingData.Poco {
xyz = "abc"
};
host.Call("Func", new
{
triggers = new CloudQueueMessage(JsonConvert.SerializeObject(trigger))
});
// Now peek at messages.
// queue name is normalized to lowercase.
var queue = account.CreateQueueClient().GetQueueReference("qname-abc");
var msgs = queue.GetMessages(10).ToArray();
Assert.Equal(1, msgs.Length);
Assert.Equal("123", msgs[0].AsString);
}
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
var storageException = e as StorageException;
var isDevStoreAccount = GetIsDevStoreAccountFromCloudStorageAccount(account.SdkObject);
if (storageException?.RequestInformation?.HttpStatusCode == 400 &&
storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue")
{
// Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'.
// If we see this error response classify the account as a premium account
account.Type = StorageAccountType.Premium;
return;
}
else if (isDevStoreAccount)
{
// If using the storage emulator, it might not be running
throw new InvalidOperationException(Constants.CheckAzureStorageEmulatorMessage, e);
}
else
{
// If not a recognized error, the credentials are invalid
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Please make sure your credentials are correct.",
account.Credentials.AccountName);
throw new InvalidOperationException(message, e);
}
}
IStorageQueueClient queueClient = account.CreateQueueClient();
IStorageQueue queue = queueClient.GetQueueReference("name");
try
{
await queue.ExistsAsync(cancellationToken);
}
catch (StorageException exception) when(IsBlobOnlyStorageException(exception))
{
account.Type = StorageAccountType.BlobOnly;
}
}
internal IStorageQueue GetQueue(QueueAttribute attrResolved, IStorageAccount account)
{
var client = account.CreateQueueClient();
string queueName = attrResolved.QueueName.ToLowerInvariant();
QueueClient.ValidateQueueName(queueName);
return(client.GetQueueReference(queueName));
}
public FakeQueueProcessor(QueueProcessorFactoryContext context, IStorageAccount storageAccount) :
base(context)
{
// map the queue names from the context to fake queues
IStorageQueueClient client = storageAccount.CreateQueueClient();
_queue = client.GetQueueReference(context.Queue.Name);
if(context.PoisonQueue != null)
{
_poisonQueue = client.GetQueueReference(context.PoisonQueue.Name);
}
}
private async Task ValidateCredentialsAsyncCore(IStorageAccount account,
bool isPrimaryAccount, CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Please make sure your credentials are correct.",
account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
if (isPrimaryAccount)
{
// Primary storage accounts require Queues
IStorageQueueClient queueClient = account.CreateQueueClient();
IStorageQueue queue = queueClient.GetQueueReference("name");
try
{
await queue.ExistsAsync(cancellationToken);
_primaryCredentials = account.Credentials;
}
catch (OperationCanceledException)
{
throw;
}
catch (StorageException exception)
{
WebException webException = exception.GetBaseException() as WebException;
if (webException != null && webException.Status == WebExceptionStatus.NameResolutionFailure)
{
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Primary storage accounts must be general "
+ "purpose accounts and not restricted blob storage accounts.", account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
throw;
}
}
}
public FakeQueueProcessor(QueueProcessorFactoryContext context, IStorageAccount storageAccount) :
base(context)
{
// map the queue names from the context to fake queues
IStorageQueueClient client = storageAccount.CreateQueueClient();
_queue = client.GetQueueReference(context.Queue.Name);
if (context.PoisonQueue != null)
{
_poisonQueue = client.GetQueueReference(context.PoisonQueue.Name);
}
}
// [Queue] has some pre-existing behavior where the storage account can be specified outside of the [Queue] attribute.
// The storage account is pulled from the ParameterInfo (which could pull in a [Storage] attribute on the container class)
// Resolve everything back down to a single attribute so we can use the binding helpers.
// This pattern should be rare since other extensions can just keep everything directly on the primary attribute.
private async Task <QueueAttribute> CollectAttributeInfo(QueueAttribute attrResolved, ParameterInfo parameter, INameResolver nameResolver)
{
// Look for [Storage] attribute and squirrel over
IStorageAccount account = await _accountProvider.GetStorageAccountAsync(parameter, CancellationToken.None, nameResolver);
StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext
{
Parameter = parameter
};
IStorageQueueClient client = account.CreateQueueClient(clientFactoryContext);
return(new ResolvedQueueAttribute(attrResolved.QueueName, client));
}
public void TestGenericSucceeds()
{
IStorageAccount account = CreateFakeStorageAccount();
var host = TestHelpers.NewJobHost <GenericProgram <ICollector <string> > >(account);
host.Call("Func");
// Now peek at messages.
var queue = account.CreateQueueClient().GetQueueReference(QueueName);
var msgs = queue.GetMessages(10).ToArray();
Assert.Equal(1, msgs.Length);
Assert.Equal("123", msgs[0].AsString);
}
public void Queue_IfBoundToCloudQueue_BindsAndCreatesQueue()
{
// Arrange
IStorageAccount account = CreateFakeStorageAccount();
IStorageQueueClient client = account.CreateQueueClient();
IStorageQueue triggerQueue = CreateQueue(client, TriggerQueueName);
triggerQueue.AddMessage(triggerQueue.CreateMessage("ignore"));
// Act
CloudQueue result = RunTrigger <CloudQueue>(account, typeof(BindToCloudQueueProgram),
(s) => BindToCloudQueueProgram.TaskSource = s);
// Assert
Assert.NotNull(result);
Assert.Equal(QueueName, result.Name);
IStorageQueue queue = client.GetQueueReference(QueueName);
Assert.True(queue.Exists());
}
public async Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
{
ParameterInfo parameter = context.Parameter;
var queueTrigger = TypeUtility.GetResolvedAttribute <QueueTriggerAttribute>(context.Parameter);
if (queueTrigger == null)
{
return(null);
}
string queueName = Resolve(queueTrigger.QueueName);
queueName = NormalizeAndValidate(queueName);
ITriggerDataArgumentBinding <IStorageQueueMessage> argumentBinding = InnerProvider.TryCreate(parameter);
if (argumentBinding == null)
{
throw new InvalidOperationException(
"Can't bind QueueTrigger to type '" + parameter.ParameterType + "'.");
}
IStorageAccount account = await _accountProvider.GetStorageAccountAsync(queueTrigger, context.CancellationToken, _nameResolver);
// requires storage account with queue support
account.AssertTypeOneOf(StorageAccountType.GeneralPurpose);
StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext
{
Parameter = parameter
};
IStorageQueueClient client = account.CreateQueueClient(clientFactoryContext);
IStorageQueue queue = client.GetQueueReference(queueName);
ITriggerBinding binding = new QueueTriggerBinding(parameter.Name, queue, argumentBinding,
_queueConfiguration, _exceptionHandler, _messageEnqueuedWatcherSetter,
_sharedContextProvider, _loggerFactory);
return(binding);
}
public async Task CloudQueueAddMessage_AddsMessage()
{
// Arrange
CloudStorageAccount sdkAccount = CreateSdkAccount();
string queueName = GetQueueName("add-message");
CloudQueue sdkQueue = CreateSdkQueue(sdkAccount, queueName);
await sdkQueue.CreateIfNotExistsAsync();
try
{
string expectedContent = "hello";
IStorageAccount product = CreateProductUnderTest(sdkAccount);
IStorageQueueClient client = product.CreateQueueClient();
Assert.NotNull(client); // Guard
IStorageQueue queue = client.GetQueueReference(queueName);
Assert.NotNull(queue); // Guard
IStorageQueueMessage message = queue.CreateMessage(expectedContent);
Assert.NotNull(message); // Guard
// Act
queue.AddMessageAsync(message, CancellationToken.None).GetAwaiter().GetResult();
// Assert
CloudQueueMessage sdkMessage = await sdkQueue.GetMessageAsync();
Assert.NotNull(sdkMessage);
Assert.Equal(expectedContent, sdkMessage.AsString);
}
finally
{
await sdkQueue.DeleteAsync();
}
}
public void Queue_IfBoundToICollectorCloudQueueMessage_AddEnqueuesMessage()
{
// Arrange
string expectedContent = Guid.NewGuid().ToString();
IStorageAccount account = CreateFakeStorageAccount();
IStorageQueueClient client = account.CreateQueueClient();
IStorageQueue triggerQueue = CreateQueue(client, TriggerQueueName);
triggerQueue.AddMessage(triggerQueue.CreateMessage(expectedContent));
// Act
RunTrigger <object>(account, typeof(BindToICollectorCloudQueueMessageProgram),
(s) => BindToICollectorCloudQueueMessageProgram.TaskSource = s);
// Assert
IStorageQueue queue = client.GetQueueReference(QueueName);
IEnumerable <IStorageQueueMessage> messages = queue.GetMessages(messageCount: 10);
Assert.NotNull(messages);
Assert.Equal(1, messages.Count());
IStorageQueueMessage message = messages.Single();
Assert.Equal(expectedContent, message.AsString);
}
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
var storageException = e as StorageException;
if (storageException?.RequestInformation?.HttpStatusCode == 400 &&
storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue")
{
// Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'.
// If we see this error response classify the account as a premium account
account.Type = StorageAccountType.Premium;
return;
}
else
{
// If not a recognized error, the credentials are invalid
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Please make sure your credentials are correct.",
account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
}
IStorageQueueClient queueClient = account.CreateQueueClient();
IStorageQueue queue = queueClient.GetQueueReference("name");
try
{
await queue.ExistsAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (StorageException exception)
{
WebException webException = exception.GetBaseException() as WebException;
if (webException?.Status == WebExceptionStatus.NameResolutionFailure)
{
// Blob-only storage accounts do not support services other than Blob.
// If we see a name resolution failure on the queue endpoint classify as a blob-only account
account.Type = StorageAccountType.BlobOnly;
return;
}
throw;
}
}
internal static async Task <JobHostContext> CreateAndLogHostStartedAsync(
IStorageAccountProvider storageAccountProvider,
IFunctionIndexProvider functionIndexProvider,
IBindingProvider bindingProvider,
IHostIdProvider hostIdProvider,
IHostInstanceLoggerProvider hostInstanceLoggerProvider,
IFunctionInstanceLoggerProvider functionInstanceLoggerProvider,
IFunctionOutputLoggerProvider functionOutputLoggerProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IConsoleProvider consoleProvider,
CancellationToken shutdownToken,
CancellationToken cancellationToken)
{
using (CancellationTokenSource combinedCancellationSource =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken))
{
CancellationToken combinedCancellationToken = combinedCancellationSource.Token;
await WriteSiteExtensionManifestAsync(combinedCancellationToken);
IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(
combinedCancellationToken);
IHostInstanceLogger hostInstanceLogger = await hostInstanceLoggerProvider.GetAsync(
combinedCancellationToken);
IFunctionInstanceLogger functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(
combinedCancellationToken);
IFunctionOutputLogger functionOutputLogger = await functionOutputLoggerProvider.GetAsync(
combinedCancellationToken);
IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken);
IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll());
FunctionExecutor executor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger,
backgroundExceptionDispatcher);
TextWriter consoleOut = consoleProvider.Out;
IFunctionExecutor hostCallExecutor;
IListener listener;
HostOutputMessage hostOutputMessage;
if (dashboardAccount != null)
{
string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken);
string sharedQueueName = HostQueueNames.GetHostQueueName(hostId);
IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient();
IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName);
IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue,
queueConfiguration, backgroundExceptionDispatcher, consoleOut, functions,
functionInstanceLogger);
Guid hostInstanceId = Guid.NewGuid();
string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N"));
IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName);
IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue,
queueConfiguration, backgroundExceptionDispatcher, consoleOut, functions,
functionInstanceLogger);
HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor
{
SharedContainerName = HostContainerNames.Hosts,
SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId,
InstanceBlobName = hostInstanceId.ToString("N"),
ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds
};
IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(
dashboardAccount,
heartbeatDescriptor.SharedContainerName,
heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName));
IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods();
Assembly hostAssembly = GetHostAssembly(indexedMethods);
string displayName = hostAssembly != null?hostAssembly.GetName().Name : "Unknown";
hostOutputMessage = new DataOnlyHostOutputMessage
{
HostInstanceId = hostInstanceId,
HostDisplayName = displayName,
SharedQueueName = sharedQueueName,
InstanceQueueName = instanceQueueName,
Heartbeat = heartbeatDescriptor,
WebJobRunIdentifier = WebJobRunIdentifier.Current
};
hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand,
backgroundExceptionDispatcher, shutdownToken, executor);
IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory,
sharedQueueListenerFactory, instanceQueueListenerFactory);
listener = CreateHostListener(hostListenerFactory, heartbeatCommand, backgroundExceptionDispatcher,
shutdownToken, executor);
// Publish this to Azure logging account so that a web dashboard can see it.
await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger,
combinedCancellationToken);
}
else
{
hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, executor);
IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory, executor);
IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener);
listener = shutdownListener;
hostOutputMessage = new DataOnlyHostOutputMessage();
}
executor.HostOutputMessage = hostOutputMessage;
IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors();
int descriptorsCount = descriptors.Count();
if (descriptorsCount == 0)
{
consoleOut.WriteLine(
"No functions found. Try making job classes and methods public.");
}
else
{
consoleOut.WriteLine("Found the following functions:");
foreach (FunctionDescriptor descriptor in descriptors)
{
consoleOut.WriteLine(descriptor.FullName);
}
}
return(new JobHostContext(functions, hostCallExecutor, listener, consoleOut));
}
}
public static async Task <JobHostContext> CreateAndLogHostStartedAsync(
JobHost host,
IStorageAccountProvider storageAccountProvider,
IQueueConfiguration queueConfiguration,
ITypeLocator typeLocator,
IJobActivator activator,
INameResolver nameResolver,
IConsoleProvider consoleProvider,
JobHostConfiguration config,
CancellationToken shutdownToken,
CancellationToken cancellationToken,
IHostIdProvider hostIdProvider = null,
FunctionExecutor functionExecutor = null,
IFunctionIndexProvider functionIndexProvider = null,
IBindingProvider bindingProvider = null,
IHostInstanceLoggerProvider hostInstanceLogerProvider = null,
IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = null,
IFunctionOutputLoggerProvider functionOutputLoggerProvider = null,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher = null,
SingletonManager singletonManager = null)
{
if (hostIdProvider == null)
{
hostIdProvider = new DynamicHostIdProvider(storageAccountProvider, () => functionIndexProvider);
}
IExtensionTypeLocator extensionTypeLocator = new ExtensionTypeLocator(typeLocator);
if (backgroundExceptionDispatcher == null)
{
backgroundExceptionDispatcher = BackgroundExceptionDispatcher.Instance;
}
ContextAccessor <IMessageEnqueuedWatcher> messageEnqueuedWatcherAccessor = new ContextAccessor <IMessageEnqueuedWatcher>();
ContextAccessor <IBlobWrittenWatcher> blobWrittenWatcherAccessor = new ContextAccessor <IBlobWrittenWatcher>();
ISharedContextProvider sharedContextProvider = new SharedContextProvider();
// Create a wrapper TraceWriter that delegates to both the user
// TraceWriter specified on Config (if present), as well as to Console
TraceWriter trace = new ConsoleTraceWriter(config.Tracing, consoleProvider.Out);
// Register system services with the service container
config.AddService <INameResolver>(nameResolver);
ExtensionConfigContext context = new ExtensionConfigContext
{
Config = config,
Trace = trace,
Host = host
};
InvokeExtensionConfigProviders(context);
IExtensionRegistry extensions = config.GetExtensions();
ITriggerBindingProvider triggerBindingProvider = DefaultTriggerBindingProvider.Create(nameResolver,
storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, backgroundExceptionDispatcher,
messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, extensions, trace);
if (bindingProvider == null)
{
bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions);
}
DefaultLoggerProvider loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace);
if (singletonManager == null)
{
singletonManager = new SingletonManager(storageAccountProvider, backgroundExceptionDispatcher, config.Singleton, trace, config.NameResolver);
}
using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken))
{
CancellationToken combinedCancellationToken = combinedCancellationSource.Token;
await WriteSiteExtensionManifestAsync(combinedCancellationToken);
IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken);
IHostInstanceLogger hostInstanceLogger = null;
if (hostInstanceLogerProvider != null)
{
hostInstanceLogger = await hostInstanceLogerProvider.GetAsync(combinedCancellationToken);
}
else
{
hostInstanceLogger = await((IHostInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken);
}
IFunctionInstanceLogger functionInstanceLogger = null;
if (functionInstanceLoggerProvider != null)
{
functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken);
}
else
{
functionInstanceLogger = (IFunctionInstanceLogger)(await((IFunctionInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken));
}
IFunctionOutputLogger functionOutputLogger = null;
if (functionOutputLoggerProvider != null)
{
functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken);
}
else
{
functionOutputLogger = (IFunctionOutputLogger)(await((IFunctionOutputLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken));
}
if (functionExecutor == null)
{
functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, backgroundExceptionDispatcher, trace, config.FunctionTimeout);
}
if (functionIndexProvider == null)
{
functionIndexProvider = new FunctionIndexProvider(typeLocator, triggerBindingProvider, bindingProvider, activator, functionExecutor, extensions, singletonManager);
}
IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken);
IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace);
IFunctionExecutor hostCallExecutor;
IListener listener;
HostOutputMessage hostOutputMessage;
if (dashboardAccount != null)
{
string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken);
string sharedQueueName = HostQueueNames.GetHostQueueName(hostId);
IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient();
IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName);
IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue,
queueConfiguration, backgroundExceptionDispatcher, trace, functions,
functionInstanceLogger, functionExecutor);
Guid hostInstanceId = Guid.NewGuid();
string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N"));
IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName);
IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue,
queueConfiguration, backgroundExceptionDispatcher, trace, functions,
functionInstanceLogger, functionExecutor);
HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor
{
SharedContainerName = HostContainerNames.Hosts,
SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId,
InstanceBlobName = hostInstanceId.ToString("N"),
ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds
};
IStorageBlockBlob blob = dashboardAccount.CreateBlobClient()
.GetContainerReference(heartbeatDescriptor.SharedContainerName)
.GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName);
IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob));
IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods();
Assembly hostAssembly = GetHostAssembly(indexedMethods);
string displayName = hostAssembly != null?hostAssembly.GetName().Name : "Unknown";
hostOutputMessage = new DataOnlyHostOutputMessage
{
HostInstanceId = hostInstanceId,
HostDisplayName = displayName,
SharedQueueName = sharedQueueName,
InstanceQueueName = instanceQueueName,
Heartbeat = heartbeatDescriptor,
WebJobRunIdentifier = WebJobRunIdentifier.Current
};
hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand,
backgroundExceptionDispatcher, shutdownToken, functionExecutor);
IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory,
sharedQueueListenerFactory, instanceQueueListenerFactory);
listener = CreateHostListener(hostListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken);
// Publish this to Azure logging account so that a web dashboard can see it.
await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken);
}
else
{
hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor);
IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory);
IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener);
listener = shutdownListener;
hostOutputMessage = new DataOnlyHostOutputMessage();
}
functionExecutor.HostOutputMessage = hostOutputMessage;
IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors();
int descriptorsCount = descriptors.Count();
if (descriptorsCount == 0)
{
trace.Warning(string.Format("No job functions found. Try making your job classes and methods public. {0}",
Constants.ExtensionInitializationMessage), TraceSource.Indexing);
}
else
{
StringBuilder functionsTrace = new StringBuilder();
functionsTrace.AppendLine("Found the following functions:");
foreach (FunctionDescriptor descriptor in descriptors)
{
functionsTrace.AppendLine(descriptor.FullName);
}
trace.Info(functionsTrace.ToString(), TraceSource.Indexing);
}
return(new JobHostContext(functions, hostCallExecutor, listener, trace));
}
}
public async Task <IListener> CreateAsync(CancellationToken cancellationToken)
{
// Note that these clients are intentionally for the storage account rather than for the dashboard account.
// We use the storage, not dashboard, account for the blob receipt container and blob trigger queues.
IStorageQueueClient primaryQueueClient = _hostAccount.CreateQueueClient();
IStorageBlobClient primaryBlobClient = _hostAccount.CreateBlobClient();
// Important: We're using the storage account of the function target here, which is the account that the
// function the listener is for is targeting. This is the account that will be used
// to read the trigger blob.
IStorageBlobClient targetBlobClient = _dataAccount.CreateBlobClient();
IStorageQueueClient targetQueueClient = _dataAccount.CreateQueueClient();
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId);
IStorageQueue hostBlobTriggerQueue = primaryQueueClient.GetQueueReference(hostBlobTriggerQueueName);
SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>(
new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter));
SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>(
new SharedBlobListenerFactory(hostId, _hostAccount, _exceptionHandler, _blobWrittenWatcherSetter));
// Register the blob container we wish to monitor with the shared blob listener.
await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient,
hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken);
// Create a "bridge" listener that will monitor the blob
// notification queue and dispatch to the target job function.
SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(_hostAccount, sharedQueueWatcher, hostBlobTriggerQueue,
_queueConfiguration, _exceptionHandler, _trace, _loggerFactory, sharedBlobListener.BlobWritterWatcher));
var queueListener = new BlobListener(sharedBlobQueueListener);
// determine which client to use for the poison queue
// by default this should target the same storage account
// as the blob container we're monitoring
var poisonQueueClient = targetQueueClient;
if (_dataAccount.Type != StorageAccountType.GeneralPurpose ||
_blobsConfiguration.CentralizedPoisonQueue)
{
// use the primary storage account if the centralize flag is true,
// or if the target storage account doesn't support queues
poisonQueueClient = primaryQueueClient;
}
// Register our function with the shared blob queue listener
RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient);
// check a flag in the shared context to see if we've created the singleton
// shared blob listener in this host instance
object singletonListenerCreated = false;
if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated))
{
// Create a singleton shared blob listener, since we only
// want a single instance of the blob poll/scan logic to be running
// across host instances
var singletonBlobListener = _singletonManager.CreateHostSingletonListener(
new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId);
_sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true);
return(new CompositeListener(singletonBlobListener, queueListener));
}
else
{
// We've already created the singleton blob listener
// so just return our queue listener. Note that while we want the
// blob scan to be singleton, the shared queue listener needs to run
// on ALL instances so load can be scaled out
return(queueListener);
}
}
// Do the full runtime intitialization. This includes static initialization.
// This mainly means:
// - indexing the functions
// - spinning up the listeners (so connecting to the services)
public static async Task <JobHostContext> CreateJobHostContextAsync(
this JobHostConfiguration config,
ServiceProviderWrapper services, // Results from first phase
JobHost host,
CancellationToken shutdownToken,
CancellationToken cancellationToken)
{
FunctionExecutor functionExecutor = services.GetService <FunctionExecutor>();
IFunctionIndexProvider functionIndexProvider = services.GetService <IFunctionIndexProvider>();
ITriggerBindingProvider triggerBindingProvider = services.GetService <ITriggerBindingProvider>();
IBindingProvider bindingProvider = services.GetService <IBindingProvider>();
SingletonManager singletonManager = services.GetService <SingletonManager>();
IJobActivator activator = services.GetService <IJobActivator>();
IHostIdProvider hostIdProvider = services.GetService <IHostIdProvider>();
INameResolver nameResolver = services.GetService <INameResolver>();
IExtensionRegistry extensions = services.GetExtensions();
IStorageAccountProvider storageAccountProvider = services.GetService <IStorageAccountProvider>();
ILoggerFactory loggerFactory = services.GetService <ILoggerFactory>();
IFunctionResultAggregatorFactory aggregatorFactory = services.GetService <IFunctionResultAggregatorFactory>();
IAsyncCollector <FunctionInstanceLogEntry> functionEventCollector = null;
// Create the aggregator if all the pieces are configured
IAsyncCollector <FunctionInstanceLogEntry> aggregator = null;
if (loggerFactory != null && aggregatorFactory != null && config.Aggregator.IsEnabled)
{
aggregator = aggregatorFactory.Create(config.Aggregator.BatchSize, config.Aggregator.FlushTimeout, loggerFactory);
}
IQueueConfiguration queueConfiguration = services.GetService <IQueueConfiguration>();
var blobsConfiguration = config.Blobs;
TraceWriter trace = services.GetService <TraceWriter>();
IAsyncCollector <FunctionInstanceLogEntry> registeredFunctionEventCollector = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >();
if (registeredFunctionEventCollector != null && aggregator != null)
{
// If there are both an aggregator and a registered FunctionEventCollector, wrap them in a composite
functionEventCollector = new CompositeFunctionEventCollector(new[] { registeredFunctionEventCollector, aggregator });
}
else
{
// Otherwise, take whichever one is null (or use null if both are)
functionEventCollector = aggregator ?? registeredFunctionEventCollector;
}
IWebJobsExceptionHandler exceptionHandler = services.GetService <IWebJobsExceptionHandler>();
if (exceptionHandler != null)
{
exceptionHandler.Initialize(host);
}
bool hasFastTableHook = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >() != null;
bool noDashboardStorage = config.DashboardConnectionString == null;
// Only testing will override these interfaces.
IHostInstanceLoggerProvider hostInstanceLoggerProvider = services.GetService <IHostInstanceLoggerProvider>();
IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = services.GetService <IFunctionInstanceLoggerProvider>();
IFunctionOutputLoggerProvider functionOutputLoggerProvider = services.GetService <IFunctionOutputLoggerProvider>();
if (hostInstanceLoggerProvider == null && functionInstanceLoggerProvider == null && functionOutputLoggerProvider == null)
{
if (hasFastTableHook && noDashboardStorage)
{
var loggerProvider = new FastTableLoggerProvider(trace);
hostInstanceLoggerProvider = loggerProvider;
functionInstanceLoggerProvider = loggerProvider;
functionOutputLoggerProvider = loggerProvider;
}
else
{
var loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace);
hostInstanceLoggerProvider = loggerProvider;
functionInstanceLoggerProvider = loggerProvider;
functionOutputLoggerProvider = loggerProvider;
}
}
using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken))
{
CancellationToken combinedCancellationToken = combinedCancellationSource.Token;
await WriteSiteExtensionManifestAsync(combinedCancellationToken);
IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken);
IHostInstanceLogger hostInstanceLogger = await hostInstanceLoggerProvider.GetAsync(combinedCancellationToken);
IFunctionInstanceLogger functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken);
IFunctionOutputLogger functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken);
if (functionExecutor == null)
{
var extensionRegistry = config.GetService <IExtensionRegistry>();
var globalFunctionFilters = extensionRegistry.GetFunctionFilters();
functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, exceptionHandler, trace, functionEventCollector, loggerFactory, globalFunctionFilters);
services.AddService(functionExecutor);
}
if (functionIndexProvider == null)
{
functionIndexProvider = new FunctionIndexProvider(
services.GetService <ITypeLocator>(),
triggerBindingProvider,
bindingProvider,
activator,
functionExecutor,
extensions,
singletonManager,
trace,
loggerFactory);
// Important to set this so that the func we passed to DynamicHostIdProvider can pick it up.
services.AddService <IFunctionIndexProvider>(functionIndexProvider);
}
IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken);
IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace, loggerFactory);
IFunctionExecutor hostCallExecutor;
IListener listener;
HostOutputMessage hostOutputMessage;
string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken);
if (string.Compare(config.HostId, hostId, StringComparison.OrdinalIgnoreCase) != 0)
{
// if this isn't a static host ID, provide the HostId on the config
// so it is accessible
config.HostId = hostId;
}
if (dashboardAccount == null)
{
hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor);
IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory);
IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener);
listener = shutdownListener;
hostOutputMessage = new DataOnlyHostOutputMessage();
}
else
{
string sharedQueueName = HostQueueNames.GetHostQueueName(hostId);
IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient();
IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName);
IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue,
queueConfiguration, exceptionHandler, trace, loggerFactory, functions,
functionInstanceLogger, functionExecutor);
Guid hostInstanceId = Guid.NewGuid();
string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N"));
IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName);
IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue,
queueConfiguration, exceptionHandler, trace, loggerFactory, functions,
functionInstanceLogger, functionExecutor);
HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor
{
SharedContainerName = HostContainerNames.Hosts,
SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId,
InstanceBlobName = hostInstanceId.ToString("N"),
ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds
};
IStorageBlockBlob blob = dashboardAccount.CreateBlobClient()
.GetContainerReference(heartbeatDescriptor.SharedContainerName)
.GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName);
IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob));
IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods();
Assembly hostAssembly = GetHostAssembly(indexedMethods);
string displayName = hostAssembly != null?AssemblyNameCache.GetName(hostAssembly).Name : "Unknown";
hostOutputMessage = new DataOnlyHostOutputMessage
{
HostInstanceId = hostInstanceId,
HostDisplayName = displayName,
SharedQueueName = sharedQueueName,
InstanceQueueName = instanceQueueName,
Heartbeat = heartbeatDescriptor,
WebJobRunIdentifier = WebJobRunIdentifier.Current
};
hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand,
exceptionHandler, shutdownToken, functionExecutor);
IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory,
sharedQueueListenerFactory, instanceQueueListenerFactory);
listener = CreateHostListener(hostListenerFactory, heartbeatCommand, exceptionHandler, shutdownToken);
// Publish this to Azure logging account so that a web dashboard can see it.
await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken);
}
functionExecutor.HostOutputMessage = hostOutputMessage;
IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors();
int descriptorsCount = descriptors.Count();
ILogger startupLogger = loggerFactory?.CreateLogger(LogCategories.Startup);
if (config.UsingDevelopmentSettings)
{
string msg = "Development settings applied";
trace.Verbose(msg);
startupLogger?.LogDebug(msg);
}
if (descriptorsCount == 0)
{
string msg = string.Format("No job functions found. Try making your job classes and methods public. {0}",
Constants.ExtensionInitializationMessage);
trace.Warning(msg, Host.TraceSource.Indexing);
startupLogger?.LogWarning(msg);
}
else
{
StringBuilder functionsTrace = new StringBuilder();
functionsTrace.AppendLine("Found the following functions:");
foreach (FunctionDescriptor descriptor in descriptors)
{
functionsTrace.AppendLine(descriptor.FullName);
}
string msg = functionsTrace.ToString();
trace.Info(msg, Host.TraceSource.Indexing);
startupLogger?.LogInformation(msg);
}
return(new JobHostContext(
functions,
hostCallExecutor,
listener,
trace,
functionEventCollector,
loggerFactory));
}
}
