public async Task ExecuteAsync_UpdatesScanInfoManager() { int testScanBlobLimitPerPoll = 6; string firstContainerName = Guid.NewGuid().ToString(); string secondContainerName = Guid.NewGuid().ToString(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer firstContainer = account.CreateBlobClient().GetContainerReference(firstContainerName); IStorageBlobContainer secondContainer = account.CreateBlobClient().GetContainerReference(secondContainerName); TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager(); string accountName = account.Credentials.AccountName; testScanInfoManager.SetScanInfo(accountName, firstContainerName, DateTime.MinValue); testScanInfoManager.SetScanInfo(accountName, secondContainerName, DateTime.MinValue); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); await product.RegisterAsync(firstContainer, executor, CancellationToken.None); await product.RegisterAsync(secondContainer, executor, CancellationToken.None); var firstExpectedNames = new List <string>(); for (int i = 0; i < 3; i++) { firstExpectedNames.Add(CreateAblobAndUploadToContainer(firstContainer)); } RunExecuteWithMultiPollingInterval(firstExpectedNames, product, executor, testScanBlobLimitPerPoll / 2); // only expect the first container to have updated its scanInfo Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]); int count; testScanInfoManager.UpdateCounts[accountName].TryGetValue(secondContainerName, out count); Assert.Equal(0, count); await Task.Delay(10); var secondExpectedNames = new List <string>(); for (int i = 0; i < 7; i++) { secondExpectedNames.Add(CreateAblobAndUploadToContainer(secondContainer)); } RunExecuteWithMultiPollingInterval(secondExpectedNames, product, executor, testScanBlobLimitPerPoll / 2); // this time, only expect the second container to have updated its scanInfo Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]); Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][secondContainerName]); }
public async Task <IListener> CreateAsync(IFunctionExecutor executor, CancellationToken cancellationToken) { SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreate <SharedQueueWatcher>( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreate <SharedBlobListener>( new SharedBlobListenerFactory(_account, _backgroundExceptionDispatcher, _blobWrittenWatcherSetter)); // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. IStorageQueueClient queueClient = _account.CreateQueueClient(); IStorageBlobClient blobClient = _account.CreateBlobClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); IStorageQueue hostBlobTriggerQueue = queueClient.GetQueueReference(hostBlobTriggerQueueName); IListener blobDiscoveryToQueueMessageListener = await CreateBlobDiscoveryToQueueMessageListenerAsync( hostId, sharedBlobListener, blobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken); IListener queueMessageToTriggerExecutionListener = CreateQueueMessageToTriggerExecutionListener(executor, _sharedContextProvider, sharedQueueWatcher, queueClient, hostBlobTriggerQueue, blobClient, sharedBlobListener.BlobWritterWatcher); IListener compositeListener = new CompositeListener( blobDiscoveryToQueueMessageListener, queueMessageToTriggerExecutionListener); return(compositeListener); }
public void TestBlobListenerWithContainerBiggerThanThreshold() { int testScanBlobLimitPerPoll = 1; string containerName = Path.GetRandomFileName(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager()); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); product.Register(container, executor); product.Start(); // populate with 5 blobs List <string> expectedNames = new List <string>(); for (int i = 0; i < 5; i++) { expectedNames.Add(CreateAblobAndUploadToContainer(container)); } RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll); // Now run again; shouldn't show up. RunExecuterWithExpectedBlobs(new List <string>(), product, executor); }
private async Task EnsureLoggersAsync(CancellationToken cancellationToken) { if (_loggersSet) { return; } IStorageAccount dashboardAccount = await _storageAccountProvider.GetDashboardAccountAsync(cancellationToken); if (dashboardAccount != null) { // Create logging against a live Azure account. IStorageBlobClient dashboardBlobClient = dashboardAccount.CreateBlobClient(); IPersistentQueueWriter <PersistentQueueMessage> queueWriter = new PersistentQueueWriter <PersistentQueueMessage>(dashboardBlobClient); PersistentQueueLogger queueLogger = new PersistentQueueLogger(queueWriter); _hostInstanceLogger = queueLogger; _functionInstanceLogger = new CompositeFunctionInstanceLogger(queueLogger, new ConsoleFunctionInstanceLogger()); _functionOutputLogger = new BlobFunctionOutputLogger(dashboardBlobClient); } else { // No auxillary logging. Logging interfaces are nops or in-memory. _hostInstanceLogger = new NullHostInstanceLogger(); _functionInstanceLogger = new ConsoleFunctionInstanceLogger(); _functionOutputLogger = new ConsoleFunctionOutputLogger(); } _loggersSet = true; }
public async Task <IBinding> TryCreateAsync(BindingProviderContext context) { ParameterInfo parameter = context.Parameter; BlobAttribute blob = parameter.GetCustomAttribute <BlobAttribute>(inherit: false); if (blob == null) { return(null); } string resolvedCombinedPath = Resolve(blob.BlobPath); IBindableBlobPath path = BindableBlobPath.Create(resolvedCombinedPath); path.ValidateContractCompatibility(context.BindingDataContract); IBlobArgumentBinding argumentBinding = _provider.TryCreate(parameter, blob.Access); if (argumentBinding == null) { throw new InvalidOperationException("Can't bind Blob to type '" + parameter.ParameterType + "'."); } IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.CancellationToken); IStorageBlobClient client = account.CreateBlobClient(); IBinding binding = new BlobBinding(parameter.Name, argumentBinding, client, path); return(binding); }
public async Task RegisterAsync_InitializesWithScanInfoManager() { string containerName = Guid.NewGuid().ToString(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName); TestBlobScanInfoManager scanInfoManager = new TestBlobScanInfoManager(); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(scanInfoManager); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); // Create a few blobs. for (int i = 0; i < 5; i++) { CreateAblobAndUploadToContainer(container); } await scanInfoManager.UpdateLatestScanAsync(account.Credentials.AccountName, containerName, DateTime.UtcNow); await product.RegisterAsync(container, executor, CancellationToken.None); // delay slightly so we guarantee a later timestamp await Task.Delay(10); var expectedNames = new List <string>(); expectedNames.Add(CreateAblobAndUploadToContainer(container)); RunExecuterWithExpectedBlobs(expectedNames, product, executor); }
public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure() { int testScanBlobLimitPerPoll = 6; string containerName = Guid.NewGuid().ToString(); // we'll introduce multiple errors to make sure we take the earliest timestamp DateTime earliestErrorTime = DateTime.UtcNow; var timeMap = new Dictionary <string, DateTimeOffset>(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName, account.CreateBlobClient(), blobs => { // Set a blob with "throw" to a specific date and time. Make sure the error blob // is earlier than the others. foreach (IStorageBlob blob in blobs.Results) { ((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name]; } }); TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager(); string accountName = account.Credentials.AccountName; testScanInfoManager.SetScanInfo(accountName, containerName, DateTime.MinValue); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); await product.RegisterAsync(container, executor, CancellationToken.None); // Induce a failure to make sure the timestamp is earlier than the failure. var expectedNames = new List <string>(); for (int i = 0; i < 7; i++) { string name; if (i % 3 == 0) { name = CreateAblobAndUploadToContainer(container, "throw"); timeMap[name] = earliestErrorTime.AddMinutes(i); } else { name = CreateAblobAndUploadToContainer(container, "test"); timeMap[name] = earliestErrorTime.AddMinutes(10); } expectedNames.Add(name); } RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll); DateTime?storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, containerName); Assert.True(storedTime < earliestErrorTime); Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][containerName]); }
private static IStorageBlob CreateBlobReference(string containerName, string blobName) { IStorageAccount account = CreateAccount(); IStorageBlobClient client = account.CreateBlobClient(); IStorageBlobContainer container = client.GetContainerReference(containerName); return(container.GetBlockBlobReference(blobName)); }
private static IStorageBlobContainer CreateContainer(IStorageAccount account, string containerName) { IStorageBlobClient client = account.CreateBlobClient(); IStorageBlobContainer container = client.GetContainerReference(containerName); container.CreateIfNotExists(); return(container); }
public void TestBlobListenerWithMultipleContainers() { int testScanBlobLimitPerPoll = 6, containerCount = 2; string firstContainerName = Path.GetRandomFileName(); string secondContainerName = Path.GetRandomFileName(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer firstContainer = account.CreateBlobClient().GetContainerReference(firstContainerName); IStorageBlobContainer secondContainer = account.CreateBlobClient().GetContainerReference(secondContainerName); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager()); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); product.Register(firstContainer, executor); product.Register(secondContainer, executor); product.Start(); // populate first container with 5 blobs > page size and second with 2 blobs < page size // page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3 List <string> firstContainerExpectedNames = new List <string>(); for (int i = 0; i < 5; i++) { firstContainerExpectedNames.Add(CreateAblobAndUploadToContainer(firstContainer)); } RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount); Thread.Sleep(10); List <string> secondContainerExpectedNames = new List <string>(); for (int i = 0; i < 2; i++) { secondContainerExpectedNames.Add(CreateAblobAndUploadToContainer(secondContainer)); } RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount); // Now run again; shouldn't show up. RunExecuterWithExpectedBlobs(new List <string>(), product, executor); }
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (Exception e) { var storageException = e as StorageException; var isDevStoreAccount = GetIsDevStoreAccountFromCloudStorageAccount(account.SdkObject); if (storageException?.RequestInformation?.HttpStatusCode == 400 && storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue") { // Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'. // If we see this error response classify the account as a premium account account.Type = StorageAccountType.Premium; return; } else if (isDevStoreAccount) { // If using the storage emulator, it might not be running throw new InvalidOperationException(Constants.CheckAzureStorageEmulatorMessage, e); } else { // If not a recognized error, the credentials are invalid string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Please make sure your credentials are correct.", account.Credentials.AccountName); throw new InvalidOperationException(message, e); } } IStorageQueueClient queueClient = account.CreateQueueClient(); IStorageQueue queue = queueClient.GetQueueReference("name"); try { await queue.ExistsAsync(cancellationToken); } catch (StorageException exception) when(IsBlobOnlyStorageException(exception)) { account.Type = StorageAccountType.BlobOnly; } }
private async Task <IStorageBlobClient> GetClientAsync( BlobAttribute blobAttribute, CancellationToken cancellationToken) { IStorageAccount account = await _accountProvider.GetStorageAccountAsync(blobAttribute, cancellationToken, _nameResolver); IStorageBlobClient client = account.CreateBlobClient(); return(client); }
private async Task ValidateCredentialsAsyncCore(IStorageAccount account, bool isPrimaryAccount, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch { string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Please make sure your credentials are correct.", account.Credentials.AccountName); throw new InvalidOperationException(message); } if (isPrimaryAccount) { // Primary storage accounts require Queues IStorageQueueClient queueClient = account.CreateQueueClient(); IStorageQueue queue = queueClient.GetQueueReference("name"); try { await queue.ExistsAsync(cancellationToken); _primaryCredentials = account.Credentials; } catch (OperationCanceledException) { throw; } catch (StorageException exception) { WebException webException = exception.GetBaseException() as WebException; if (webException != null && webException.Status == WebExceptionStatus.NameResolutionFailure) { string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Primary storage accounts must be general " + "purpose accounts and not restricted blob storage accounts.", account.Credentials.AccountName); throw new InvalidOperationException(message); } throw; } } }
protected override IStorageBlobContainer GetContainer(string accountName) { // Get the container via a full connection string Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; // singleton requires block blobs, cannot be premium storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly); IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); var container = blobClient.GetContainerReference(HostContainerNames.Hosts); return(container); }
public async Task <IBinding> TryCreateAsync(BindingProviderContext context) { ParameterInfo parameter = context.Parameter; BlobAttribute blobAttribute = parameter.GetCustomAttribute <BlobAttribute>(inherit: false); if (blobAttribute == null) { return(null); } string resolvedPath = Resolve(blobAttribute.BlobPath); IBindableBlobPath path = null; IStorageAccount account = await _accountProvider.GetStorageAccountAsync(context.Parameter, context.CancellationToken); StorageClientFactoryContext clientFactoryContext = new StorageClientFactoryContext { Parameter = context.Parameter }; IStorageBlobClient client = account.CreateBlobClient(clientFactoryContext); // first try to bind to the Container IArgumentBinding <IStorageBlobContainer> containerArgumentBinding = _blobContainerArgumentProvider.TryCreate(parameter); if (containerArgumentBinding == null) { // if this isn't a Container binding, try a Blob binding IBlobArgumentBinding blobArgumentBinding = _blobArgumentProvider.TryCreate(parameter, blobAttribute.Access); if (blobArgumentBinding == null) { throw new InvalidOperationException("Can't bind Blob to type '" + parameter.ParameterType + "'."); } path = BindableBlobPath.Create(resolvedPath); path.ValidateContractCompatibility(context.BindingDataContract); return(new BlobBinding(parameter.Name, blobArgumentBinding, client, path)); } path = BindableBlobPath.Create(resolvedPath, isContainerBinding: true); path.ValidateContractCompatibility(context.BindingDataContract); BlobContainerBinding.ValidateContainerBinding(blobAttribute, parameter.ParameterType, path); return(new BlobContainerBinding(parameter.Name, containerArgumentBinding, client, path)); }
public void BlobPolling_IgnoresClockSkew() { int testScanBlobLimitPerPoll = 3; string containerName = Path.GetRandomFileName(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobClient client = account.CreateBlobClient(); var now = DateTimeOffset.UtcNow; var timeMap = new Dictionary <string, DateTimeOffset>(); IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName, client, (IStorageBlobResultSegment blobs) => { // Simulate some extreme clock skew -- the first one's LastUpdated // wll be 60 seconds ago and the second will be 59 seconds ago. foreach (IStorageBlob blob in blobs.Results) { ((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name]; } }); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager()); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); product.Register(container, executor); product.Start(); List <string> expectedNames = new List <string>(); expectedNames.Add(CreateAblobAndUploadToContainer(container)); timeMap[expectedNames.Single()] = now.AddSeconds(-60); RunExecuterWithExpectedBlobs(expectedNames, product, executor); expectedNames.Clear(); expectedNames.Add(CreateAblobAndUploadToContainer(container)); timeMap[expectedNames.Single()] = now.AddSeconds(-59); // We should see the new item. RunExecuterWithExpectedBlobs(expectedNames, product, executor); }
public void TestBlobListener() { const string containerName = "container"; IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName); IBlobListenerStrategy product = new ScanContainersStrategy(); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); product.Register(container, executor); product.Start(); executor.ExecuteLambda = (_) => { throw new InvalidOperationException("shouldn't be any blobs in the container"); }; product.Execute(); const string expectedBlobName = "foo1.csv"; IStorageBlockBlob blob = container.GetBlockBlobReference(expectedBlobName); container.CreateIfNotExists(); blob.UploadText("ignore"); int count = 0; executor.ExecuteLambda = (b) => { count++; Assert.Equal(expectedBlobName, b.Name); return(true); }; product.Execute(); Assert.Equal(1, count); // Now run again; shouldn't show up. executor.ExecuteLambda = (_) => { throw new InvalidOperationException("shouldn't retrigger the same blob"); }; product.Execute(); }
internal IStorageBlobDirectory GetLockDirectory(string accountName) { if (string.IsNullOrEmpty(accountName)) { accountName = ConnectionStringNames.Storage; } IStorageBlobDirectory storageDirectory = null; if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory)) { Task <IStorageAccount> task = _accountProvider.GetAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _lockDirectoryMap[accountName] = storageDirectory; } return(storageDirectory); }
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue() { // Arrange IStorageAccount account = CreateAccount(); IStorageBlobClient client = account.CreateBlobClient(); string containerName = "container"; IStorageBlobContainer container = client.GetContainerReference(containerName); IStorageBlobContainer otherContainer = client.GetContainerReference("other"); IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}"); ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input); IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch"); // Act Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None); // Assert Assert.True(task.Result); }
public BlobTriggerBinding(ParameterInfo parameter, IStorageAccount hostAccount, IStorageAccount dataAccount, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, JobHostBlobsConfiguration blobsConfiguration, IWebJobsExceptionHandler exceptionHandler, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, SingletonManager singletonManager, ILoggerFactory loggerFactory) { _parameter = parameter ?? throw new ArgumentNullException(nameof(parameter)); _hostAccount = hostAccount ?? throw new ArgumentNullException(nameof(hostAccount)); _dataAccount = dataAccount ?? throw new ArgumentNullException(nameof(dataAccount)); StorageClientFactoryContext context = new StorageClientFactoryContext { Parameter = parameter }; _blobClient = dataAccount.CreateBlobClient(context); _accountName = BlobClient.GetAccountName(_blobClient); _path = path ?? throw new ArgumentNullException(nameof(path)); _hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider)); _queueConfiguration = queueConfiguration ?? throw new ArgumentNullException(nameof(queueConfiguration)); _blobsConfiguration = blobsConfiguration ?? throw new ArgumentNullException(nameof(blobsConfiguration)); _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); _blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter)); _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter)); _sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider)); _singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager)); _loggerFactory = loggerFactory; _converter = CreateConverter(_blobClient); _bindingDataContract = CreateBindingDataContract(path); }
public void ScanBlobScanLogHybridPollingStrategyTestBlobListener() { string containerName = Path.GetRandomFileName(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager()); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); product.Register(container, executor); product.Start(); RunExecuterWithExpectedBlobs(new List <string>(), product, executor); string expectedBlobName = CreateAblobAndUploadToContainer(container); RunExecuterWithExpectedBlobs(new List <string>() { expectedBlobName }, product, executor); // Now run again; shouldn't show up. RunExecuterWithExpectedBlobs(new List <string>(), product, executor); }
internal IStorageBlobDirectory GetLockDirectory(string accountName) { if (string.IsNullOrEmpty(accountName)) { accountName = ConnectionStringNames.Storage; } IStorageBlobDirectory storageDirectory = null; if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory)) { Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None); IStorageAccount storageAccount = task.Result; // singleton requires block blobs, cannot be premium storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly); IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts) .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _lockDirectoryMap[accountName] = storageDirectory; } return(storageDirectory); }
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch { string message = String.Format(CultureInfo.CurrentCulture, "The account credentials for '{0}' are incorrect.", account.Credentials.AccountName); throw new InvalidOperationException(message); } }
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch { string message = String.Format(CultureInfo.CurrentCulture, "The account credentials for '{0}' are incorrect.", account.Credentials.AccountName); throw new InvalidOperationException(message); } }
public BlobTriggerBinding(string parameterName, IArgumentBinding <IStorageBlob> argumentBinding, IStorageAccount account, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, TextWriter log) { if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (account == null) { throw new ArgumentNullException("account"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (log == null) { throw new ArgumentNullException("log"); } _parameterName = parameterName; _argumentBinding = argumentBinding; _account = account; _client = account.CreateBlobClient(); _accountName = BlobClient.GetAccountName(_client); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _log = log; _converter = CreateConverter(_client); _bindingDataContract = CreateBindingDataContract(path); }
private static IStorageBlobContainer GetContainerReference(IStorageAccount account, string containerName) { IStorageBlobClient client = account.CreateBlobClient(); return(client.GetContainerReference(ContainerName)); }
public BlobTriggerBinding(ParameterInfo parameter, IArgumentBinding<IStorageBlob> argumentBinding, IStorageAccount hostAccount, IStorageAccount dataAccount, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, TraceWriter trace) { if (parameter == null) { throw new ArgumentNullException("parameter"); } if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (hostAccount == null) { throw new ArgumentNullException("hostAccount"); } if (dataAccount == null) { throw new ArgumentNullException("dataAccount"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (trace == null) { throw new ArgumentNullException("trace"); } _parameter = parameter; _argumentBinding = argumentBinding; _hostAccount = hostAccount; _dataAccount = dataAccount; StorageClientFactoryContext context = new StorageClientFactoryContext { Parameter = parameter }; _blobClient = dataAccount.CreateBlobClient(context); _accountName = BlobClient.GetAccountName(_blobClient); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _trace = trace; _converter = CreateConverter(_blobClient); _bindingDataContract = CreateBindingDataContract(path); }
public static async Task <JobHostContext> CreateAndLogHostStartedAsync( JobHost host, IStorageAccountProvider storageAccountProvider, IQueueConfiguration queueConfiguration, ITypeLocator typeLocator, IJobActivator activator, INameResolver nameResolver, IConsoleProvider consoleProvider, JobHostConfiguration config, CancellationToken shutdownToken, CancellationToken cancellationToken, IHostIdProvider hostIdProvider = null, FunctionExecutor functionExecutor = null, IFunctionIndexProvider functionIndexProvider = null, IBindingProvider bindingProvider = null, IHostInstanceLoggerProvider hostInstanceLogerProvider = null, IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = null, IFunctionOutputLoggerProvider functionOutputLoggerProvider = null, IBackgroundExceptionDispatcher backgroundExceptionDispatcher = null, SingletonManager singletonManager = null) { if (hostIdProvider == null) { hostIdProvider = new DynamicHostIdProvider(storageAccountProvider, () => functionIndexProvider); } IExtensionTypeLocator extensionTypeLocator = new ExtensionTypeLocator(typeLocator); if (backgroundExceptionDispatcher == null) { backgroundExceptionDispatcher = BackgroundExceptionDispatcher.Instance; } ContextAccessor <IMessageEnqueuedWatcher> messageEnqueuedWatcherAccessor = new ContextAccessor <IMessageEnqueuedWatcher>(); ContextAccessor <IBlobWrittenWatcher> blobWrittenWatcherAccessor = new ContextAccessor <IBlobWrittenWatcher>(); ISharedContextProvider sharedContextProvider = new SharedContextProvider(); // Create a wrapper TraceWriter that delegates to both the user // TraceWriter specified on Config (if present), as well as to Console TraceWriter trace = new ConsoleTraceWriter(config.Tracing, consoleProvider.Out); // Register system services with the service container config.AddService <INameResolver>(nameResolver); ExtensionConfigContext context = new ExtensionConfigContext { Config = config, Trace = trace, Host = host }; InvokeExtensionConfigProviders(context); IExtensionRegistry extensions = config.GetExtensions(); ITriggerBindingProvider triggerBindingProvider = DefaultTriggerBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, backgroundExceptionDispatcher, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, extensions, trace); if (bindingProvider == null) { bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions); } DefaultLoggerProvider loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace); if (singletonManager == null) { singletonManager = new SingletonManager(storageAccountProvider, backgroundExceptionDispatcher, config.Singleton, trace, config.NameResolver); } using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken)) { CancellationToken combinedCancellationToken = combinedCancellationSource.Token; await WriteSiteExtensionManifestAsync(combinedCancellationToken); IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken); IHostInstanceLogger hostInstanceLogger = null; if (hostInstanceLogerProvider != null) { hostInstanceLogger = await hostInstanceLogerProvider.GetAsync(combinedCancellationToken); } else { hostInstanceLogger = await((IHostInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken); } IFunctionInstanceLogger functionInstanceLogger = null; if (functionInstanceLoggerProvider != null) { functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken); } else { functionInstanceLogger = (IFunctionInstanceLogger)(await((IFunctionInstanceLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken)); } IFunctionOutputLogger functionOutputLogger = null; if (functionOutputLoggerProvider != null) { functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken); } else { functionOutputLogger = (IFunctionOutputLogger)(await((IFunctionOutputLoggerProvider)loggerProvider).GetAsync(combinedCancellationToken)); } if (functionExecutor == null) { functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, backgroundExceptionDispatcher, trace, config.FunctionTimeout); } if (functionIndexProvider == null) { functionIndexProvider = new FunctionIndexProvider(typeLocator, triggerBindingProvider, bindingProvider, activator, functionExecutor, extensions, singletonManager); } IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken); IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace); IFunctionExecutor hostCallExecutor; IListener listener; HostOutputMessage hostOutputMessage; if (dashboardAccount != null) { string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken); string sharedQueueName = HostQueueNames.GetHostQueueName(hostId); IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient(); IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName); IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue, queueConfiguration, backgroundExceptionDispatcher, trace, functions, functionInstanceLogger, functionExecutor); Guid hostInstanceId = Guid.NewGuid(); string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N")); IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName); IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue, queueConfiguration, backgroundExceptionDispatcher, trace, functions, functionInstanceLogger, functionExecutor); HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor { SharedContainerName = HostContainerNames.Hosts, SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId, InstanceBlobName = hostInstanceId.ToString("N"), ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds }; IStorageBlockBlob blob = dashboardAccount.CreateBlobClient() .GetContainerReference(heartbeatDescriptor.SharedContainerName) .GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName); IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob)); IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods(); Assembly hostAssembly = GetHostAssembly(indexedMethods); string displayName = hostAssembly != null?hostAssembly.GetName().Name : "Unknown"; hostOutputMessage = new DataOnlyHostOutputMessage { HostInstanceId = hostInstanceId, HostDisplayName = displayName, SharedQueueName = sharedQueueName, InstanceQueueName = instanceQueueName, Heartbeat = heartbeatDescriptor, WebJobRunIdentifier = WebJobRunIdentifier.Current }; hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken, functionExecutor); IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory, sharedQueueListenerFactory, instanceQueueListenerFactory); listener = CreateHostListener(hostListenerFactory, heartbeatCommand, backgroundExceptionDispatcher, shutdownToken); // Publish this to Azure logging account so that a web dashboard can see it. await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken); } else { hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor); IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory); IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener); listener = shutdownListener; hostOutputMessage = new DataOnlyHostOutputMessage(); } functionExecutor.HostOutputMessage = hostOutputMessage; IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors(); int descriptorsCount = descriptors.Count(); if (descriptorsCount == 0) { trace.Warning(string.Format("No job functions found. Try making your job classes and methods public. {0}", Constants.ExtensionInitializationMessage), TraceSource.Indexing); } else { StringBuilder functionsTrace = new StringBuilder(); functionsTrace.AppendLine("Found the following functions:"); foreach (FunctionDescriptor descriptor in descriptors) { functionsTrace.AppendLine(descriptor.FullName); } trace.Info(functionsTrace.ToString(), TraceSource.Indexing); } return(new JobHostContext(functions, hostCallExecutor, listener, trace)); } }
public BlobTriggerBinding(string parameterName, IArgumentBinding<IStorageBlob> argumentBinding, IStorageAccount account, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, TextWriter log) { if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (account == null) { throw new ArgumentNullException("account"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (log == null) { throw new ArgumentNullException("log"); } _parameterName = parameterName; _argumentBinding = argumentBinding; _account = account; _client = account.CreateBlobClient(); _accountName = BlobClient.GetAccountName(_client); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _log = log; _converter = CreateConverter(_client); _bindingDataContract = CreateBindingDataContract(path); }
/// <summary>Initializes a new instance of the <see cref="HeartbeatCommand"/> class.</summary> /// <param name="account">The storage account in which to write the heartbeat.</param> /// <param name="containerName">The name of the container in which to write the heartbeat.</param> /// <param name="blobName">The name of the heartbeat blob (including the directory name, if any).</param> public HeartbeatCommand(IStorageAccount account, string containerName, string blobName) : this(account.CreateBlobClient().GetContainerReference(containerName).GetBlockBlobReference(blobName)) { }
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. IStorageBlobClient client = account.CreateBlobClient(); try { // This can hang for a long time if the account name is wrong. // If will fail fast if the password is incorrect. await client.GetServicePropertiesAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (Exception e) { var storageException = e as StorageException; if (storageException?.RequestInformation?.HttpStatusCode == 400 && storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue") { // Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'. // If we see this error response classify the account as a premium account account.Type = StorageAccountType.Premium; return; } else { // If not a recognized error, the credentials are invalid string message = String.Format(CultureInfo.CurrentCulture, "Invalid storage account '{0}'. Please make sure your credentials are correct.", account.Credentials.AccountName); throw new InvalidOperationException(message); } } IStorageQueueClient queueClient = account.CreateQueueClient(); IStorageQueue queue = queueClient.GetQueueReference("name"); try { await queue.ExistsAsync(cancellationToken); } catch (OperationCanceledException) { throw; } catch (StorageException exception) { WebException webException = exception.GetBaseException() as WebException; if (webException?.Status == WebExceptionStatus.NameResolutionFailure) { // Blob-only storage accounts do not support services other than Blob. // If we see a name resolution failure on the queue endpoint classify as a blob-only account account.Type = StorageAccountType.BlobOnly; return; } throw; } }
// Do the full runtime intitialization. This includes static initialization. // This mainly means: // - indexing the functions // - spinning up the listeners (so connecting to the services) public static async Task <JobHostContext> CreateJobHostContextAsync( this JobHostConfiguration config, ServiceProviderWrapper services, // Results from first phase JobHost host, CancellationToken shutdownToken, CancellationToken cancellationToken) { FunctionExecutor functionExecutor = services.GetService <FunctionExecutor>(); IFunctionIndexProvider functionIndexProvider = services.GetService <IFunctionIndexProvider>(); ITriggerBindingProvider triggerBindingProvider = services.GetService <ITriggerBindingProvider>(); IBindingProvider bindingProvider = services.GetService <IBindingProvider>(); SingletonManager singletonManager = services.GetService <SingletonManager>(); IJobActivator activator = services.GetService <IJobActivator>(); IHostIdProvider hostIdProvider = services.GetService <IHostIdProvider>(); INameResolver nameResolver = services.GetService <INameResolver>(); IExtensionRegistry extensions = services.GetExtensions(); IStorageAccountProvider storageAccountProvider = services.GetService <IStorageAccountProvider>(); ILoggerFactory loggerFactory = services.GetService <ILoggerFactory>(); IFunctionResultAggregatorFactory aggregatorFactory = services.GetService <IFunctionResultAggregatorFactory>(); IAsyncCollector <FunctionInstanceLogEntry> functionEventCollector = null; // Create the aggregator if all the pieces are configured IAsyncCollector <FunctionInstanceLogEntry> aggregator = null; if (loggerFactory != null && aggregatorFactory != null && config.Aggregator.IsEnabled) { aggregator = aggregatorFactory.Create(config.Aggregator.BatchSize, config.Aggregator.FlushTimeout, loggerFactory); } IQueueConfiguration queueConfiguration = services.GetService <IQueueConfiguration>(); var blobsConfiguration = config.Blobs; TraceWriter trace = services.GetService <TraceWriter>(); IAsyncCollector <FunctionInstanceLogEntry> registeredFunctionEventCollector = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >(); if (registeredFunctionEventCollector != null && aggregator != null) { // If there are both an aggregator and a registered FunctionEventCollector, wrap them in a composite functionEventCollector = new CompositeFunctionEventCollector(new[] { registeredFunctionEventCollector, aggregator }); } else { // Otherwise, take whichever one is null (or use null if both are) functionEventCollector = aggregator ?? registeredFunctionEventCollector; } IWebJobsExceptionHandler exceptionHandler = services.GetService <IWebJobsExceptionHandler>(); if (exceptionHandler != null) { exceptionHandler.Initialize(host); } bool hasFastTableHook = services.GetService <IAsyncCollector <FunctionInstanceLogEntry> >() != null; bool noDashboardStorage = config.DashboardConnectionString == null; // Only testing will override these interfaces. IHostInstanceLoggerProvider hostInstanceLoggerProvider = services.GetService <IHostInstanceLoggerProvider>(); IFunctionInstanceLoggerProvider functionInstanceLoggerProvider = services.GetService <IFunctionInstanceLoggerProvider>(); IFunctionOutputLoggerProvider functionOutputLoggerProvider = services.GetService <IFunctionOutputLoggerProvider>(); if (hostInstanceLoggerProvider == null && functionInstanceLoggerProvider == null && functionOutputLoggerProvider == null) { if (hasFastTableHook && noDashboardStorage) { var loggerProvider = new FastTableLoggerProvider(trace); hostInstanceLoggerProvider = loggerProvider; functionInstanceLoggerProvider = loggerProvider; functionOutputLoggerProvider = loggerProvider; } else { var loggerProvider = new DefaultLoggerProvider(storageAccountProvider, trace); hostInstanceLoggerProvider = loggerProvider; functionInstanceLoggerProvider = loggerProvider; functionOutputLoggerProvider = loggerProvider; } } using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken)) { CancellationToken combinedCancellationToken = combinedCancellationSource.Token; await WriteSiteExtensionManifestAsync(combinedCancellationToken); IStorageAccount dashboardAccount = await storageAccountProvider.GetDashboardAccountAsync(combinedCancellationToken); IHostInstanceLogger hostInstanceLogger = await hostInstanceLoggerProvider.GetAsync(combinedCancellationToken); IFunctionInstanceLogger functionInstanceLogger = await functionInstanceLoggerProvider.GetAsync(combinedCancellationToken); IFunctionOutputLogger functionOutputLogger = await functionOutputLoggerProvider.GetAsync(combinedCancellationToken); if (functionExecutor == null) { var extensionRegistry = config.GetService <IExtensionRegistry>(); var globalFunctionFilters = extensionRegistry.GetFunctionFilters(); functionExecutor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, exceptionHandler, trace, functionEventCollector, loggerFactory, globalFunctionFilters); services.AddService(functionExecutor); } if (functionIndexProvider == null) { functionIndexProvider = new FunctionIndexProvider( services.GetService <ITypeLocator>(), triggerBindingProvider, bindingProvider, activator, functionExecutor, extensions, singletonManager, trace, loggerFactory); // Important to set this so that the func we passed to DynamicHostIdProvider can pick it up. services.AddService <IFunctionIndexProvider>(functionIndexProvider); } IFunctionIndex functions = await functionIndexProvider.GetAsync(combinedCancellationToken); IListenerFactory functionsListenerFactory = new HostListenerFactory(functions.ReadAll(), singletonManager, activator, nameResolver, trace, loggerFactory); IFunctionExecutor hostCallExecutor; IListener listener; HostOutputMessage hostOutputMessage; string hostId = await hostIdProvider.GetHostIdAsync(cancellationToken); if (string.Compare(config.HostId, hostId, StringComparison.OrdinalIgnoreCase) != 0) { // if this isn't a static host ID, provide the HostId on the config // so it is accessible config.HostId = hostId; } if (dashboardAccount == null) { hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor); IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory); IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener); listener = shutdownListener; hostOutputMessage = new DataOnlyHostOutputMessage(); } else { string sharedQueueName = HostQueueNames.GetHostQueueName(hostId); IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient(); IStorageQueue sharedQueue = dashboardQueueClient.GetQueueReference(sharedQueueName); IListenerFactory sharedQueueListenerFactory = new HostMessageListenerFactory(sharedQueue, queueConfiguration, exceptionHandler, trace, loggerFactory, functions, functionInstanceLogger, functionExecutor); Guid hostInstanceId = Guid.NewGuid(); string instanceQueueName = HostQueueNames.GetHostQueueName(hostInstanceId.ToString("N")); IStorageQueue instanceQueue = dashboardQueueClient.GetQueueReference(instanceQueueName); IListenerFactory instanceQueueListenerFactory = new HostMessageListenerFactory(instanceQueue, queueConfiguration, exceptionHandler, trace, loggerFactory, functions, functionInstanceLogger, functionExecutor); HeartbeatDescriptor heartbeatDescriptor = new HeartbeatDescriptor { SharedContainerName = HostContainerNames.Hosts, SharedDirectoryName = HostDirectoryNames.Heartbeats + "/" + hostId, InstanceBlobName = hostInstanceId.ToString("N"), ExpirationInSeconds = (int)HeartbeatIntervals.ExpirationInterval.TotalSeconds }; IStorageBlockBlob blob = dashboardAccount.CreateBlobClient() .GetContainerReference(heartbeatDescriptor.SharedContainerName) .GetBlockBlobReference(heartbeatDescriptor.SharedDirectoryName + "/" + heartbeatDescriptor.InstanceBlobName); IRecurrentCommand heartbeatCommand = new UpdateHostHeartbeatCommand(new HeartbeatCommand(blob)); IEnumerable <MethodInfo> indexedMethods = functions.ReadAllMethods(); Assembly hostAssembly = GetHostAssembly(indexedMethods); string displayName = hostAssembly != null?AssemblyNameCache.GetName(hostAssembly).Name : "Unknown"; hostOutputMessage = new DataOnlyHostOutputMessage { HostInstanceId = hostInstanceId, HostDisplayName = displayName, SharedQueueName = sharedQueueName, InstanceQueueName = instanceQueueName, Heartbeat = heartbeatDescriptor, WebJobRunIdentifier = WebJobRunIdentifier.Current }; hostCallExecutor = CreateHostCallExecutor(instanceQueueListenerFactory, heartbeatCommand, exceptionHandler, shutdownToken, functionExecutor); IListenerFactory hostListenerFactory = new CompositeListenerFactory(functionsListenerFactory, sharedQueueListenerFactory, instanceQueueListenerFactory); listener = CreateHostListener(hostListenerFactory, heartbeatCommand, exceptionHandler, shutdownToken); // Publish this to Azure logging account so that a web dashboard can see it. await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken); } functionExecutor.HostOutputMessage = hostOutputMessage; IEnumerable <FunctionDescriptor> descriptors = functions.ReadAllDescriptors(); int descriptorsCount = descriptors.Count(); ILogger startupLogger = loggerFactory?.CreateLogger(LogCategories.Startup); if (config.UsingDevelopmentSettings) { string msg = "Development settings applied"; trace.Verbose(msg); startupLogger?.LogDebug(msg); } if (descriptorsCount == 0) { string msg = string.Format("No job functions found. Try making your job classes and methods public. {0}", Constants.ExtensionInitializationMessage); trace.Warning(msg, Host.TraceSource.Indexing); startupLogger?.LogWarning(msg); } else { StringBuilder functionsTrace = new StringBuilder(); functionsTrace.AppendLine("Found the following functions:"); foreach (FunctionDescriptor descriptor in descriptors) { functionsTrace.AppendLine(descriptor.FullName); } string msg = functionsTrace.ToString(); trace.Info(msg, Host.TraceSource.Indexing); startupLogger?.LogInformation(msg); } return(new JobHostContext( functions, hostCallExecutor, listener, trace, functionEventCollector, loggerFactory)); } }
public BlobTriggerBinding(ParameterInfo parameter, IArgumentBinding <IStorageBlob> argumentBinding, IStorageAccount hostAccount, IStorageAccount dataAccount, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter, IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter, ISharedContextProvider sharedContextProvider, SingletonManager singletonManager, TraceWriter trace) { if (parameter == null) { throw new ArgumentNullException("parameter"); } if (argumentBinding == null) { throw new ArgumentNullException("argumentBinding"); } if (hostAccount == null) { throw new ArgumentNullException("hostAccount"); } if (dataAccount == null) { throw new ArgumentNullException("dataAccount"); } if (path == null) { throw new ArgumentNullException("path"); } if (hostIdProvider == null) { throw new ArgumentNullException("hostIdProvider"); } if (queueConfiguration == null) { throw new ArgumentNullException("queueConfiguration"); } if (backgroundExceptionDispatcher == null) { throw new ArgumentNullException("backgroundExceptionDispatcher"); } if (blobWrittenWatcherSetter == null) { throw new ArgumentNullException("blobWrittenWatcherSetter"); } if (messageEnqueuedWatcherSetter == null) { throw new ArgumentNullException("messageEnqueuedWatcherSetter"); } if (sharedContextProvider == null) { throw new ArgumentNullException("sharedContextProvider"); } if (singletonManager == null) { throw new ArgumentNullException("singletonManager"); } if (trace == null) { throw new ArgumentNullException("trace"); } _parameter = parameter; _argumentBinding = argumentBinding; _hostAccount = hostAccount; _dataAccount = dataAccount; StorageClientFactoryContext context = new StorageClientFactoryContext { Parameter = parameter }; _blobClient = dataAccount.CreateBlobClient(context); _accountName = BlobClient.GetAccountName(_blobClient); _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; _sharedContextProvider = sharedContextProvider; _singletonManager = singletonManager; _trace = trace; _converter = CreateConverter(_blobClient); _bindingDataContract = CreateBindingDataContract(path); }
/// <summary>Initializes a new instance of the <see cref="HeartbeatCommand"/> class.</summary> /// <param name="account">The storage account in which to write the heartbeat.</param> /// <param name="containerName">The name of the container in which to write the heartbeat.</param> /// <param name="blobName">The name of the heartbeat blob (including the directory name, if any).</param> public HeartbeatCommand(IStorageAccount account, string containerName, string blobName) : this(account.CreateBlobClient().GetContainerReference(containerName).GetBlockBlobReference(blobName)) { }