public async Task <IListener> CreateAsync(IFunctionExecutor executor, CancellationToken cancellationToken) { SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreate <SharedQueueWatcher>( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreate <SharedBlobListener>( new SharedBlobListenerFactory(_account, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter)); // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. IStorageQueueClient queueClient = _account.CreateQueueClient(); IStorageBlobClient blobClient = _account.CreateBlobClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); IStorageQueue hostBlobTriggerQueue = queueClient.GetQueueReference(hostBlobTriggerQueueName); IListener blobDiscoveryToQueueMessageListener = await CreateBlobDiscoveryToQueueMessageListenerAsync( hostId, sharedBlobListener, blobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken); IListener queueMessageToTriggerExecutionListener = CreateQueueMessageToTriggerExecutionListener(executor, _sharedContextProvider, sharedQueueWatcher, queueClient, hostBlobTriggerQueue, blobClient, sharedBlobListener.BlobWritterWatcher); IListener compositeListener = new CompositeListener( blobDiscoveryToQueueMessageListener, queueMessageToTriggerExecutionListener); return(compositeListener); }
private static IStorageQueue CreateQueue(IStorageAccount account, string queueName) { IStorageQueueClient client = account.CreateQueueClient(); IStorageQueue queue = client.GetQueueReference(queueName); queue.CreateIfNotExists(); return queue; }
public async Task CloudQueueCreate_IfNotExist_CreatesQueue() { // Arrange CloudStorageAccount sdkAccount = CreateSdkAccount(); string queueName = GetQueueName("create-queue"); CloudQueue sdkQueue = CreateSdkQueue(sdkAccount, queueName); try { IStorageAccount product = CreateProductUnderTest(sdkAccount); IStorageQueueClient client = product.CreateQueueClient(); Assert.NotNull(client); // Guard IStorageQueue queue = client.GetQueueReference(queueName); Assert.NotNull(queue); // Guard // Act await queue.CreateIfNotExistsAsync(CancellationToken.None); // Assert Assert.True(await sdkQueue.ExistsAsync()); } finally { if (await sdkQueue.ExistsAsync()) { await sdkQueue.DeleteAsync(); } } }
public async Task <IBinding> TryCreateAsync(BindingProviderContext context) { ParameterInfo parameter = context.Parameter; QueueAttribute queueAttribute = parameter.GetCustomAttribute <QueueAttribute>(inherit: false); if (queueAttribute == null) { return(null); } string queueName = Resolve(queueAttribute.QueueName); IBindableQueuePath path = BindableQueuePath.Create(queueName); path.ValidateContractCompatibility(context.BindingDataContract); IArgumentBinding <IStorageQueue> argumentBinding = _innerProvider.TryCreate(parameter); if (argumentBinding == null) { throw new InvalidOperationException("Can't bind Queue to type '" + parameter.ParameterType + "'."); } IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.Parameter, context.CancellationToken); StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext { Parameter = parameter }; IStorageQueueClient client = account.CreateQueueClient(clientFactoryContext); IBinding binding = new QueueBinding(parameter.Name, argumentBinding, client, path); return(binding); }
public async Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context) { ParameterInfo parameter = context.Parameter; QueueTriggerAttribute queueTrigger = parameter.GetCustomAttribute <QueueTriggerAttribute>(inherit: false); if (queueTrigger == null) { return(null); } string queueName = Resolve(queueTrigger.QueueName); queueName = NormalizeAndValidate(queueName); ITriggerDataArgumentBinding <IStorageQueueMessage> argumentBinding = _innerProvider.TryCreate(parameter); if (argumentBinding == null) { throw new InvalidOperationException( "Can't bind QueueTrigger to type '" + parameter.ParameterType + "'."); } IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.CancellationToken); IStorageQueueClient client = account.CreateQueueClient(); IStorageQueue queue = client.GetQueueReference(queueName); ITriggerBinding binding = new QueueTriggerBinding(parameter.Name, queue, argumentBinding, _queueConfiguration, _backgroundExceptionDispatcher, _messageEnqueuedWatcherSetter, _sharedContextProvider, _log); return(binding); }
public void InvokeWithBindingData() { // Verify that queue binding pattern has uppercase letters in it. These get normalized to lowercase. Assert.NotEqual(ProgramWithTriggerAndBindingData.QueueOutName, ProgramWithTriggerAndBindingData.QueueOutName.ToLower()); IStorageAccount account = CreateFakeStorageAccount(); var host = TestHelpers.NewJobHost <ProgramWithTriggerAndBindingData>(account); var trigger = new ProgramWithTriggerAndBindingData.Poco { xyz = "abc" }; host.Call("Func", new { triggers = new CloudQueueMessage(JsonConvert.SerializeObject(trigger)) }); // Now peek at messages. // queue name is normalized to lowercase. var queue = account.CreateQueueClient().GetQueueReference("qname-abc"); var msgs = queue.GetMessages(10).ToArray(); Assert.Equal(1, msgs.Length); Assert.Equal("123", msgs[0].AsString); }
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (Exception e) { var storageException = e as StorageException; var isDevStoreAccount = GetIsDevStoreAccountFromCloudStorageAccount(account.SdkObject); if (storageException?.RequestInformation?.HttpStatusCode == 400 && storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue") { // Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'. // If we see this error response classify the account as a premium account account.Type = StorageAccountType.Premium; return; } else if (isDevStoreAccount) { // If using the storage emulator, it might not be running throw new InvalidOperationException(Constants.CheckAzureStorageEmulatorMessage, e); } else { // If not a recognized error, the credentials are invalid string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Please make sure your credentials are correct.", account.Credentials.AccountName); throw new InvalidOperationException(message, e); } } IStorageQueueClient queueClient = account.CreateQueueClient(); IStorageQueue queue = queueClient.GetQueueReference("name"); try { await queue.ExistsAsync(cancellationToken); } catch (StorageException exception) when(IsBlobOnlyStorageException(exception)) { account.Type = StorageAccountType.BlobOnly; } }
internal IStorageQueue GetQueue(QueueAttribute attrResolved, IStorageAccount account) { var client = account.CreateQueueClient(); string queueName = attrResolved.QueueName.ToLowerInvariant(); QueueClient.ValidateQueueName(queueName); return(client.GetQueueReference(queueName)); }
public FakeQueueProcessor(QueueProcessorFactoryContext context, IStorageAccount storageAccount) : base(context) { // map the queue names from the context to fake queues IStorageQueueClient client = storageAccount.CreateQueueClient(); _queue = client.GetQueueReference(context.Queue.Name); if(context.PoisonQueue != null) { _poisonQueue = client.GetQueueReference(context.PoisonQueue.Name); } }
private async Task ValidateCredentialsAsyncCore(IStorageAccount account, bool isPrimaryAccount, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch { string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Please make sure your credentials are correct.", account.Credentials.AccountName); throw new InvalidOperationException(message); } if (isPrimaryAccount) { // Primary storage accounts require Queues IStorageQueueClient queueClient = account.CreateQueueClient(); IStorageQueue queue = queueClient.GetQueueReference("name"); try { await queue.ExistsAsync(cancellationToken); _primaryCredentials = account.Credentials; } catch (OperationCanceledException) { throw; } catch (StorageException exception) { WebException webException = exception.GetBaseException() as WebException; if (webException != null && webException.Status == WebExceptionStatus.NameResolutionFailure) { string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Primary storage accounts must be general " + "purpose accounts and not restricted blob storage accounts.", account.Credentials.AccountName); throw new InvalidOperationException(message); } throw; } } }
public FakeQueueProcessor(QueueProcessorFactoryContext context, IStorageAccount storageAccount) : base(context) { // map the queue names from the context to fake queues IStorageQueueClient client = storageAccount.CreateQueueClient(); _queue = client.GetQueueReference(context.Queue.Name); if (context.PoisonQueue != null) { _poisonQueue = client.GetQueueReference(context.PoisonQueue.Name); } }
// [Queue] has some pre-existing behavior where the storage account can be specified outside of the [Queue] attribute. // The storage account is pulled from the ParameterInfo (which could pull in a [Storage] attribute on the container class) // Resolve everything back down to a single attribute so we can use the binding helpers. // This pattern should be rare since other extensions can just keep everything directly on the primary attribute. private async Task <QueueAttribute> CollectAttributeInfo(QueueAttribute attrResolved, ParameterInfo parameter, INameResolver nameResolver) { // Look for [Storage] attribute and squirrel over IStorageAccount account = await _accountProvider.GetStorageAccountAsync(parameter, CancellationToken.None, nameResolver); StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext { Parameter = parameter }; IStorageQueueClient client = account.CreateQueueClient(clientFactoryContext); return(new ResolvedQueueAttribute(attrResolved.QueueName, client)); }
public void TestGenericSucceeds() { IStorageAccount account = CreateFakeStorageAccount(); var host = TestHelpers.NewJobHost <GenericProgram <ICollector <string> > >(account); host.Call("Func"); // Now peek at messages. var queue = account.CreateQueueClient().GetQueueReference(QueueName); var msgs = queue.GetMessages(10).ToArray(); Assert.Equal(1, msgs.Length); Assert.Equal("123", msgs[0].AsString); }
public void Queue_IfBoundToCloudQueue_BindsAndCreatesQueue() { // Arrange IStorageAccount account = CreateFakeStorageAccount(); IStorageQueueClient client = account.CreateQueueClient(); IStorageQueue triggerQueue = CreateQueue(client, TriggerQueueName); triggerQueue.AddMessage(triggerQueue.CreateMessage("ignore")); // Act CloudQueue result = RunTrigger <CloudQueue>(account, typeof(BindToCloudQueueProgram), (s) => BindToCloudQueueProgram.TaskSource = s); // Assert Assert.NotNull(result); Assert.Equal(QueueName, result.Name); IStorageQueue queue = client.GetQueueReference(QueueName); Assert.True(queue.Exists()); }
public async Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context) { ParameterInfo parameter = context.Parameter; var queueTrigger = TypeUtility.GetResolvedAttribute <QueueTriggerAttribute>(context.Parameter); if (queueTrigger == null) { return(null); } string queueName = Resolve(queueTrigger.QueueName); queueName = NormalizeAndValidate(queueName); ITriggerDataArgumentBinding <IStorageQueueMessage> argumentBinding = InnerProvider.TryCreate(parameter); if (argumentBinding == null) { throw new InvalidOperationException( "Can't bind QueueTrigger to type '" + parameter.ParameterType + "'."); } IStorageAccount account = await _accountProvider.GetStorageAccountAsync(queueTrigger, context.CancellationToken, _nameResolver); // requires storage account with queue support account.AssertTypeOneOf(StorageAccountType.GeneralPurpose); StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext { Parameter = parameter }; IStorageQueueClient client = account.CreateQueueClient(clientFactoryContext); IStorageQueue queue = client.GetQueueReference(queueName); ITriggerBinding binding = new QueueTriggerBinding(parameter.Name, queue, argumentBinding, _queueConfiguration, _exceptionHandler, _messageEnqueuedWatcherSetter, _sharedContextProvider, _loggerFactory); return(binding); }
public async Task CloudQueueAddMessage_AddsMessage() { // Arrange CloudStorageAccount sdkAccount = CreateSdkAccount(); string queueName = GetQueueName("add-message"); CloudQueue sdkQueue = CreateSdkQueue(sdkAccount, queueName); await sdkQueue.CreateIfNotExistsAsync(); try { string expectedContent = "hello"; IStorageAccount product = CreateProductUnderTest(sdkAccount); IStorageQueueClient client = product.CreateQueueClient(); Assert.NotNull(client); // Guard IStorageQueue queue = client.GetQueueReference(queueName); Assert.NotNull(queue); // Guard IStorageQueueMessage message = queue.CreateMessage(expectedContent); Assert.NotNull(message); // Guard // Act queue.AddMessageAsync(message, CancellationToken.None).GetAwaiter().GetResult(); // Assert CloudQueueMessage sdkMessage = await sdkQueue.GetMessageAsync(); Assert.NotNull(sdkMessage); Assert.Equal(expectedContent, sdkMessage.AsString); } finally { await sdkQueue.DeleteAsync(); } }
public void Queue_IfBoundToICollectorCloudQueueMessage_AddEnqueuesMessage() { // Arrange string expectedContent = Guid.NewGuid().ToString(); IStorageAccount account = CreateFakeStorageAccount(); IStorageQueueClient client = account.CreateQueueClient(); IStorageQueue triggerQueue = CreateQueue(client, TriggerQueueName); triggerQueue.AddMessage(triggerQueue.CreateMessage(expectedContent)); // Act RunTrigger <object>(account, typeof(BindToICollectorCloudQueueMessageProgram), (s) => BindToICollectorCloudQueueMessageProgram.TaskSource = s); // Assert IStorageQueue queue = client.GetQueueReference(QueueName); IEnumerable <IStorageQueueMessage> messages = queue.GetMessages(messageCount: 10); Assert.NotNull(messages); Assert.Equal(1, messages.Count()); IStorageQueueMessage message = messages.Single(); Assert.Equal(expectedContent, message.AsString); }
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (Exception e) { var storageException = e as StorageException; if (storageException?.RequestInformation?.HttpStatusCode == 400 && storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue") { // Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'. // If we see this error response classify the account as a premium account account.Type = StorageAccountType.Premium; return; } else { // If not a recognized error, the credentials are invalid string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Please make sure your credentials are correct.", account.Credentials.AccountName); throw new InvalidOperationException(message); } } IStorageQueueClient queueClient = account.CreateQueueClient(); IStorageQueue queue = queueClient.GetQueueReference("name"); try { await queue.ExistsAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (StorageException exception) { WebException webException = exception.GetBaseException() as WebException; if (webException?.Status == WebExceptionStatus.NameResolutionFailure) { // Blob-only storage accounts do not support services other than Blob. // If we see a name resolution failure on the queue endpoint classify as a blob-only account account.Type = StorageAccountType.BlobOnly; return; } throw; } }
internal static async Task <JobHostContext> CreateAndLogHostStartedAsync( IStorageAccountProvider storageAccountProvider, IFunctionIndexProvider functionIndexProvider, IBindingProvider bindingProvider, IHostIdProvider hostIdProvider, IHostInstanceLoggerProvider hostInstanceLoggerProvider, IFunctionInstanceLoggerProvider functionInstanceLoggerProvider, IFunctionOutputLoggerProvider functionOutputLoggerProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IConsoleProvider consoleProvider, CancellationToken shutdownToken, CancellationToken cancellationToken) { using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken)) { CancellationToken combinedCancellationToken = combinedCancellationSource.Token; await WriteSiteExtensionManifestAsync(combinedCancellationToken); IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync( combinedCancellationToken); IHostInstanceLogger hostInstanceLogger = await hostInstanceLoggerProvider.GetAsync( combinedCancellationToken); IFunctionInstanceLogger functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync( combinedCancellationToken); IFunctionOutputLogger functionOutputLogger = await functionOutputLoggerProvider.GetAsync( combinedCancellationToken); IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken); IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll()); FunctionExecutor executor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, backgroundExceptionDispatcher); TextWriter consoleOut = consoleProvider.Out; IFunctionExecutor hostCallExecutor; IListener listener; HostOutputMessage hostOutputMessage; if (dashboardAccount != null) { string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken); string sharedQueueName = HostQueueNames.GetHostQueueName(hostId); IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient(); IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName); IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue, queueConfiguration, backgroundExceptionDispatcher, consoleOut, functions, functionInstanceLogger); Guid hostInstanceId = Guid.NewGuid(); string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N")); IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName); IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue, queueConfiguration, backgroundExceptionDispatcher, consoleOut, functions, functionInstanceLogger); HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor { SharedContainerName = HostContainerNames.Hosts, SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId, InstanceBlobName = hostInstanceId.ToString("N"), ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds }; IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand( dashboardAccount, heartbeatDescriptor.SharedContainerName, heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName)); IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods(); Assembly hostAssembly = GetHostAssembly(indexedMethods); string displayName = hostAssembly != null?hostAssembly.GetName().Name : "Unknown"; hostOutputMessage = new DataOnlyHostOutputMessage { HostInstanceId = hostInstanceId, HostDisplayName = displayName, SharedQueueName = sharedQueueName, InstanceQueueName = instanceQueueName, Heartbeat = heartbeatDescriptor, WebJobRunIdentifier = WebJobRunIdentifier.Current }; hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken, executor); IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory, sharedQueueListenerFactory, instanceQueueListenerFactory); listener = CreateHostListener(hostListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken, executor); // Publish this to Azure logging account so that a web dashboard can see it. await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken); } else { hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, executor); IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory, executor); IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener); listener = shutdownListener; hostOutputMessage = new DataOnlyHostOutputMessage(); } executor.HostOutputMessage = hostOutputMessage; IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors(); int descriptorsCount = descriptors.Count(); if (descriptorsCount == 0) { consoleOut.WriteLine( "No functions found. Try making job classes and methods public."); } else { consoleOut.WriteLine("Found the following functions:"); foreach (FunctionDescriptor descriptor in descriptors) { consoleOut.WriteLine(descriptor.FullName); } } return(new JobHostContext(functions, hostCallExecutor, listener, consoleOut)); } }
public static async Task <JobHostContext> CreateAndLogHostStartedAsync( JobHost host, IStorageAccountProvider storageAccountProvider, IQueueConfiguration queueConfiguration, ITypeLocator typeLocator, IJobActivator activator, INameResolver nameResolver, IConsoleProvider consoleProvider, JobHostConfiguration config, CancellationToken shutdownToken, CancellationToken cancellationToken, IHostIdProvider hostIdProvider = null, FunctionExecutor functionExecutor = null, IFunctionIndexProvider functionIndexProvider = null, IBindingProvider bindingProvider = null, IHostInstanceLoggerProvider hostInstanceLogerProvider = null, IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = null, IFunctionOutputLoggerProvider functionOutputLoggerProvider = null, IBackgroundExceptionDispatcher backgroundExceptionDispatcher = null, SingletonManager singletonManager = null) { if (hostIdProvider == null) { hostIdProvider = new DynamicHostIdProvider(storageAccountProvider, () => functionIndexProvider); } IExtensionTypeLocator extensionTypeLocator = new ExtensionTypeLocator(typeLocator); if (backgroundExceptionDispatcher == null) { backgroundExceptionDispatcher = BackgroundExceptionDispatcher.Instance; } ContextAccessor <IMessageEnqueuedWatcher> messageEnqueuedWatcherAccessor = new ContextAccessor <IMessageEnqueuedWatcher>(); ContextAccessor <IBlobWrittenWatcher> blobWrittenWatcherAccessor = new ContextAccessor <IBlobWrittenWatcher>(); ISharedContextProvider sharedContextProvider = new SharedContextProvider(); // Create a wrapper TraceWriter that delegates to both the user // TraceWriter specified on Config (if present), as well as to Console TraceWriter trace = new ConsoleTraceWriter(config.Tracing, consoleProvider.Out); // Register system services with the service container config.AddService <INameResolver>(nameResolver); ExtensionConfigContext context = new ExtensionConfigContext { Config = config, Trace = trace, Host = host }; InvokeExtensionConfigProviders(context); IExtensionRegistry extensions = config.GetExtensions(); ITriggerBindingProvider triggerBindingProvider = DefaultTriggerBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, backgroundExceptionDispatcher, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, extensions, trace); if (bindingProvider == null) { bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions); } DefaultLoggerProvider loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace); if (singletonManager == null) { singletonManager = new SingletonManager(storageAccountProvider, backgroundExceptionDispatcher, config.Singleton, trace, config.NameResolver); } using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken)) { CancellationToken combinedCancellationToken = combinedCancellationSource.Token; await WriteSiteExtensionManifestAsync(combinedCancellationToken); IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken); IHostInstanceLogger hostInstanceLogger = null; if (hostInstanceLogerProvider != null) { hostInstanceLogger = await hostInstanceLogerProvider.GetAsync(combinedCancellationToken); } else { hostInstanceLogger = await((IHostInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken); } IFunctionInstanceLogger functionInstanceLogger = null; if (functionInstanceLoggerProvider != null) { functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken); } else { functionInstanceLogger = (IFunctionInstanceLogger)(await((IFunctionInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken)); } IFunctionOutputLogger functionOutputLogger = null; if (functionOutputLoggerProvider != null) { functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken); } else { functionOutputLogger = (IFunctionOutputLogger)(await((IFunctionOutputLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken)); } if (functionExecutor == null) { functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, backgroundExceptionDispatcher, trace, config.FunctionTimeout); } if (functionIndexProvider == null) { functionIndexProvider = new FunctionIndexProvider(typeLocator, triggerBindingProvider, bindingProvider, activator, functionExecutor, extensions, singletonManager); } IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken); IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace); IFunctionExecutor hostCallExecutor; IListener listener; HostOutputMessage hostOutputMessage; if (dashboardAccount != null) { string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken); string sharedQueueName = HostQueueNames.GetHostQueueName(hostId); IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient(); IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName); IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue, queueConfiguration, backgroundExceptionDispatcher, trace, functions, functionInstanceLogger, functionExecutor); Guid hostInstanceId = Guid.NewGuid(); string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N")); IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName); IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue, queueConfiguration, backgroundExceptionDispatcher, trace, functions, functionInstanceLogger, functionExecutor); HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor { SharedContainerName = HostContainerNames.Hosts, SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId, InstanceBlobName = hostInstanceId.ToString("N"), ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds }; IStorageBlockBlob blob = dashboardAccount.CreateBlobClient() .GetContainerReference(heartbeatDescriptor.SharedContainerName) .GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName); IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob)); IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods(); Assembly hostAssembly = GetHostAssembly(indexedMethods); string displayName = hostAssembly != null?hostAssembly.GetName().Name : "Unknown"; hostOutputMessage = new DataOnlyHostOutputMessage { HostInstanceId = hostInstanceId, HostDisplayName = displayName, SharedQueueName = sharedQueueName, InstanceQueueName = instanceQueueName, Heartbeat = heartbeatDescriptor, WebJobRunIdentifier = WebJobRunIdentifier.Current }; hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken, functionExecutor); IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory, sharedQueueListenerFactory, instanceQueueListenerFactory); listener = CreateHostListener(hostListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken); // Publish this to Azure logging account so that a web dashboard can see it. await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken); } else { hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor); IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory); IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener); listener = shutdownListener; hostOutputMessage = new DataOnlyHostOutputMessage(); } functionExecutor.HostOutputMessage = hostOutputMessage; IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors(); int descriptorsCount = descriptors.Count(); if (descriptorsCount == 0) { trace.Warning(string.Format("No job functions found. Try making your job classes and methods public. {0}", Constants.ExtensionInitializationMessage), TraceSource.Indexing); } else { StringBuilder functionsTrace = new StringBuilder(); functionsTrace.AppendLine("Found the following functions:"); foreach (FunctionDescriptor descriptor in descriptors) { functionsTrace.AppendLine(descriptor.FullName); } trace.Info(functionsTrace.ToString(), TraceSource.Indexing); } return(new JobHostContext(functions, hostCallExecutor, listener, trace)); } }
public async Task <IListener> CreateAsync(CancellationToken cancellationToken) { // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. IStorageQueueClient primaryQueueClient = _hostAccount.CreateQueueClient(); IStorageBlobClient primaryBlobClient = _hostAccount.CreateBlobClient(); // Important: We're using the storage account of the function target here, which is the account that the // function the listener is for is targeting. This is the account that will be used // to read the trigger blob. IStorageBlobClient targetBlobClient = _dataAccount.CreateBlobClient(); IStorageQueueClient targetQueueClient = _dataAccount.CreateQueueClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); IStorageQueue hostBlobTriggerQueue = primaryQueueClient.GetQueueReference(hostBlobTriggerQueueName); SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>( new SharedBlobListenerFactory(hostId, _hostAccount, _exceptionHandler, _blobWrittenWatcherSetter)); // Register the blob container we wish to monitor with the shared blob listener. await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken); // Create a "bridge" listener that will monitor the blob // notification queue and dispatch to the target job function. SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>( new SharedBlobQueueListenerFactory(_hostAccount, sharedQueueWatcher, hostBlobTriggerQueue, _queueConfiguration, _exceptionHandler, _trace, _loggerFactory, sharedBlobListener.BlobWritterWatcher)); var queueListener = new BlobListener(sharedBlobQueueListener); // determine which client to use for the poison queue // by default this should target the same storage account // as the blob container we're monitoring var poisonQueueClient = targetQueueClient; if (_dataAccount.Type != StorageAccountType.GeneralPurpose || _blobsConfiguration.CentralizedPoisonQueue) { // use the primary storage account if the centralize flag is true, // or if the target storage account doesn't support queues poisonQueueClient = primaryQueueClient; } // Register our function with the shared blob queue listener RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient); // check a flag in the shared context to see if we've created the singleton // shared blob listener in this host instance object singletonListenerCreated = false; if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated)) { // Create a singleton shared blob listener, since we only // want a single instance of the blob poll/scan logic to be running // across host instances var singletonBlobListener = _singletonManager.CreateHostSingletonListener( new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId); _sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true); return(new CompositeListener(singletonBlobListener, queueListener)); } else { // We've already created the singleton blob listener // so just return our queue listener. Note that while we want the // blob scan to be singleton, the shared queue listener needs to run // on ALL instances so load can be scaled out return(queueListener); } }
// Do the full runtime intitialization. This includes static initialization. // This mainly means: // - indexing the functions // - spinning up the listeners (so connecting to the services) public static async Task <JobHostContext> CreateJobHostContextAsync( this JobHostConfiguration config, ServiceProviderWrapper services, // Results from first phase JobHost host, CancellationToken shutdownToken, CancellationToken cancellationToken) { FunctionExecutor functionExecutor = services.GetService <FunctionExecutor>(); IFunctionIndexProvider functionIndexProvider = services.GetService <IFunctionIndexProvider>(); ITriggerBindingProvider triggerBindingProvider = services.GetService <ITriggerBindingProvider>(); IBindingProvider bindingProvider = services.GetService <IBindingProvider>(); SingletonManager singletonManager = services.GetService <SingletonManager>(); IJobActivator activator = services.GetService <IJobActivator>(); IHostIdProvider hostIdProvider = services.GetService <IHostIdProvider>(); INameResolver nameResolver = services.GetService <INameResolver>(); IExtensionRegistry extensions = services.GetExtensions(); IStorageAccountProvider storageAccountProvider = services.GetService <IStorageAccountProvider>(); ILoggerFactory loggerFactory = services.GetService <ILoggerFactory>(); IFunctionResultAggregatorFactory aggregatorFactory = services.GetService <IFunctionResultAggregatorFactory>(); IAsyncCollector <FunctionInstanceLogEntry> functionEventCollector = null; // Create the aggregator if all the pieces are configured IAsyncCollector <FunctionInstanceLogEntry> aggregator = null; if (loggerFactory != null && aggregatorFactory != null && config.Aggregator.IsEnabled) { aggregator = aggregatorFactory.Create(config.Aggregator.BatchSize, config.Aggregator.FlushTimeout, loggerFactory); } IQueueConfiguration queueConfiguration = services.GetService <IQueueConfiguration>(); var blobsConfiguration = config.Blobs; TraceWriter trace = services.GetService <TraceWriter>(); IAsyncCollector <FunctionInstanceLogEntry> registeredFunctionEventCollector = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >(); if (registeredFunctionEventCollector != null && aggregator != null) { // If there are both an aggregator and a registered FunctionEventCollector, wrap them in a composite functionEventCollector = new CompositeFunctionEventCollector(new[] { registeredFunctionEventCollector, aggregator }); } else { // Otherwise, take whichever one is null (or use null if both are) functionEventCollector = aggregator ?? registeredFunctionEventCollector; } IWebJobsExceptionHandler exceptionHandler = services.GetService <IWebJobsExceptionHandler>(); if (exceptionHandler != null) { exceptionHandler.Initialize(host); } bool hasFastTableHook = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >() != null; bool noDashboardStorage = config.DashboardConnectionString == null; // Only testing will override these interfaces. IHostInstanceLoggerProvider hostInstanceLoggerProvider = services.GetService <IHostInstanceLoggerProvider>(); IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = services.GetService <IFunctionInstanceLoggerProvider>(); IFunctionOutputLoggerProvider functionOutputLoggerProvider = services.GetService <IFunctionOutputLoggerProvider>(); if (hostInstanceLoggerProvider == null && functionInstanceLoggerProvider == null && functionOutputLoggerProvider == null) { if (hasFastTableHook && noDashboardStorage) { var loggerProvider = new FastTableLoggerProvider(trace); hostInstanceLoggerProvider = loggerProvider; functionInstanceLoggerProvider = loggerProvider; functionOutputLoggerProvider = loggerProvider; } else { var loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace); hostInstanceLoggerProvider = loggerProvider; functionInstanceLoggerProvider = loggerProvider; functionOutputLoggerProvider = loggerProvider; } } using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken)) { CancellationToken combinedCancellationToken = combinedCancellationSource.Token; await WriteSiteExtensionManifestAsync(combinedCancellationToken); IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken); IHostInstanceLogger hostInstanceLogger = await hostInstanceLoggerProvider.GetAsync(combinedCancellationToken); IFunctionInstanceLogger functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken); IFunctionOutputLogger functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken); if (functionExecutor == null) { var extensionRegistry = config.GetService <IExtensionRegistry>(); var globalFunctionFilters = extensionRegistry.GetFunctionFilters(); functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, exceptionHandler, trace, functionEventCollector, loggerFactory, globalFunctionFilters); services.AddService(functionExecutor); } if (functionIndexProvider == null) { functionIndexProvider = new FunctionIndexProvider( services.GetService <ITypeLocator>(), triggerBindingProvider, bindingProvider, activator, functionExecutor, extensions, singletonManager, trace, loggerFactory); // Important to set this so that the func we passed to DynamicHostIdProvider can pick it up. services.AddService <IFunctionIndexProvider>(functionIndexProvider); } IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken); IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace, loggerFactory); IFunctionExecutor hostCallExecutor; IListener listener; HostOutputMessage hostOutputMessage; string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken); if (string.Compare(config.HostId, hostId, StringComparison.OrdinalIgnoreCase) != 0) { // if this isn't a static host ID, provide the HostId on the config // so it is accessible config.HostId = hostId; } if (dashboardAccount == null) { hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor); IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory); IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener); listener = shutdownListener; hostOutputMessage = new DataOnlyHostOutputMessage(); } else { string sharedQueueName = HostQueueNames.GetHostQueueName(hostId); IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient(); IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName); IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue, queueConfiguration, exceptionHandler, trace, loggerFactory, functions, functionInstanceLogger, functionExecutor); Guid hostInstanceId = Guid.NewGuid(); string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N")); IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName); IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue, queueConfiguration, exceptionHandler, trace, loggerFactory, functions, functionInstanceLogger, functionExecutor); HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor { SharedContainerName = HostContainerNames.Hosts, SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId, InstanceBlobName = hostInstanceId.ToString("N"), ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds }; IStorageBlockBlob blob = dashboardAccount.CreateBlobClient() .GetContainerReference(heartbeatDescriptor.SharedContainerName) .GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName); IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob)); IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods(); Assembly hostAssembly = GetHostAssembly(indexedMethods); string displayName = hostAssembly != null?AssemblyNameCache.GetName(hostAssembly).Name : "Unknown"; hostOutputMessage = new DataOnlyHostOutputMessage { HostInstanceId = hostInstanceId, HostDisplayName = displayName, SharedQueueName = sharedQueueName, InstanceQueueName = instanceQueueName, Heartbeat = heartbeatDescriptor, WebJobRunIdentifier = WebJobRunIdentifier.Current }; hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand, exceptionHandler, shutdownToken, functionExecutor); IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory, sharedQueueListenerFactory, instanceQueueListenerFactory); listener = CreateHostListener(hostListenerFactory, heartbeatCommand, exceptionHandler, shutdownToken); // Publish this to Azure logging account so that a web dashboard can see it. await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken); } functionExecutor.HostOutputMessage = hostOutputMessage; IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors(); int descriptorsCount = descriptors.Count(); ILogger startupLogger = loggerFactory?.CreateLogger(LogCategories.Startup); if (config.UsingDevelopmentSettings) { string msg = "Development settings applied"; trace.Verbose(msg); startupLogger?.LogDebug(msg); } if (descriptorsCount == 0) { string msg = string.Format("No job functions found. Try making your job classes and methods public. {0}", Constants.ExtensionInitializationMessage); trace.Warning(msg, Host.TraceSource.Indexing); startupLogger?.LogWarning(msg); } else { StringBuilder functionsTrace = new StringBuilder(); functionsTrace.AppendLine("Found the following functions:"); foreach (FunctionDescriptor descriptor in descriptors) { functionsTrace.AppendLine(descriptor.FullName); } string msg = functionsTrace.ToString(); trace.Info(msg, Host.TraceSource.Indexing); startupLogger?.LogInformation(msg); } return(new JobHostContext( functions, hostCallExecutor, listener, trace, functionEventCollector, loggerFactory)); } }