public async Task SingletonFunction_HostScope_InvocationsAreSerialized()
        {
            JobHost host = CreateTestJobHost(1);

            host.Start();

            MethodInfo  methodA = typeof(TestJobs).GetMethod("SingletonJobA_HostScope");
            List <Task> tasks   = new List <Task>();

            for (int i = 0; i < 5; i++)
            {
                tasks.Add(host.CallAsync(methodA, new { }));
            }

            MethodInfo methodB = typeof(TestJobs).GetMethod("SingletonJobB_HostScope");

            for (int i = 0; i < 5; i++)
            {
                tasks.Add(host.CallAsync(methodB, new { }));
            }
            await Task.WhenAll(tasks);

            Assert.False(TestJobs.FailureDetected);
            Assert.Equal(10, TestJobs.JobInvocations[1]);

            await VerifyLeaseState(methodA, SingletonScope.Host, "TestValue", LeaseState.Available, LeaseStatus.Unlocked);
            await VerifyLeaseState(methodB, SingletonScope.Host, "TestValue", LeaseState.Available, LeaseStatus.Unlocked);

            host.Stop();
            host.Dispose();
        }
        public async Task SingletonFunction_Exception_LeaseReleasedImmediately()
        {
            JobHost host = CreateTestJobHost(1);

            host.Start();

            WorkItem workItem = new WorkItem
            {
                ID          = 1,
                Region      = "Central",
                Zone        = 3,
                Category    = -1,
                Description = "Test Work Item"
            };

            Exception  exception = null;
            MethodInfo method    = typeof(TestJobs).GetMethod("SingletonJob");

            try
            {
                await host.CallAsync(method, new { workItem = workItem });
            }
            catch (Exception ex)
            {
                exception = ex;
            }

            Assert.Equal("Exception while executing function: TestJobs.SingletonJob", exception.Message);
            await VerifyLeaseState(method, SingletonScope.Function, "TestValue", LeaseState.Available, LeaseStatus.Unlocked);

            host.Stop();
            host.Dispose();
        }
        public async Task SingletonNonTriggeredFunction_MultipleConcurrentInvocations_InvocationsAreSerialized()
        {
            JobHost host = CreateTestJobHost(1);

            host.Start();

            // make a bunch of parallel invocations
            int         numInvocations = 20;
            List <Task> invokeTasks    = new List <Task>();
            MethodInfo  method         = typeof(TestJobs).GetMethod("SingletonJob");

            for (int i = 0; i < numInvocations; i++)
            {
                int zone = _rand.Next(3) + 1;

                WorkItem workItem = new WorkItem
                {
                    ID          = i + 1,
                    Region      = "Central",
                    Zone        = zone,
                    Category    = 3,
                    Description = "Test Work Item " + i
                };
                invokeTasks.Add(host.CallAsync(method, new { workItem = workItem }));
            }
            await Task.WhenAll(invokeTasks.ToArray());

            Assert.False(TestJobs.FailureDetected);
            Assert.Equal(numInvocations, TestJobs.JobInvocations[1]);

            await VerifyLeaseState(method, SingletonScope.Function, "TestValue", LeaseState.Available, LeaseStatus.Unlocked);

            host.Stop();
            host.Dispose();
        }
        public void IndexingExceptions_CanBeHandledByLogger()
        {
            JobHostConfiguration config = new JobHostConfiguration();

            config.TypeLocator = new FakeTypeLocator(typeof(BindingErrorsProgram));
            FunctionErrorLogger errorLogger = new FunctionErrorLogger("TestCategory");

            config.AddService <IWebJobsExceptionHandler>(new TestExceptionHandler());

            Mock <ILoggerProvider> mockProvider = new Mock <ILoggerProvider>(MockBehavior.Strict);

            mockProvider
            .Setup(m => m.CreateLogger(It.IsAny <string>()))
            .Returns(errorLogger);

            ILoggerFactory factory = new LoggerFactory();

            factory.AddProvider(mockProvider.Object);

            config.LoggerFactory = factory;

            JobHost host = new JobHost(config);

            host.Start();

            // verify the handled binding error
            FunctionIndexingException fex = errorLogger.Errors.SingleOrDefault() as FunctionIndexingException;

            Assert.True(fex.Handled);
            Assert.Equal("BindingErrorsProgram.Invalid", fex.MethodName);

            // verify that the binding error was logged
            var messages = errorLogger.GetLogMessages();

            Assert.Equal(5, messages.Count);
            LogMessage logMessage = messages.ElementAt(0);

            Assert.Equal("Error indexing method 'BindingErrorsProgram.Invalid'", logMessage.FormattedMessage);
            Assert.Same(fex, logMessage.Exception);
            Assert.Equal("Invalid container name: invalid$=+1", logMessage.Exception.InnerException.Message);
            logMessage = messages.ElementAt(1);
            Assert.Equal("Function 'BindingErrorsProgram.Invalid' failed indexing and will be disabled.", logMessage.FormattedMessage);
            Assert.Equal(Extensions.Logging.LogLevel.Warning, logMessage.Level);

            // verify that the valid function was still indexed
            logMessage = messages.ElementAt(2);
            Assert.True(logMessage.FormattedMessage.Contains("Found the following functions"));
            Assert.True(logMessage.FormattedMessage.Contains("BindingErrorsProgram.Valid"));

            // verify that the job host was started successfully
            logMessage = messages.ElementAt(4);
            Assert.Equal("Job host started", logMessage.FormattedMessage);

            host.Stop();
            host.Dispose();
        }
 protected virtual void Dispose(bool disposing)
 {
     if (disposing)
     {
         if (_jobHost != null)
         {
             _jobHost.Dispose();
             _jobHost = null;
         }
     }
 }
Exemple #6
0
        public virtual async Task DisposeAsync()
        {
            if (JobHost != null)
            {
                await JobHost.StopAsync();

                await Host.StopAsync();

                JobHost.Dispose();
            }
            Environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, string.Empty);
        }
Exemple #7
0
        public FunctionWithoutInvocationsTestsFixture()
            : base(cleanStorageAccount: true)
        {
            JobHostConfiguration hostConfiguration = new JobHostConfiguration(StorageAccount.ConnectionString)
            {
                TypeLocator = new ExplicitTypeLocator(typeof(SingleFunction))
            };

            JobHost host = new JobHost(hostConfiguration);

            host.Start();
            host.Dispose();
        }
Exemple #8
0
        public void Dispose()
        {
            Thread.Sleep(TimeSpan.FromSeconds(1)); // let the functions finish

            // this should not be needed, but for some reason some connections stay open / references keep exist after JobHost instance is stopped and disposed
            var mqttExtensionConfigProvider = _jobHost.Services.GetService(typeof(IMqttConnectionFactory)) as MqttConnectionFactory;

            mqttExtensionConfigProvider.DisconnectAll().Wait();

            _jobHost.Stop();
            _jobHost.Dispose();
            _jobHost = null;
        }
        public async Task SingletonFunction_HostScope()
        {
            JobHost host = CreateTestJobHost(1);

            host.Start();

            MethodInfo method = typeof(TestJobs).GetMethod("SingletonJobA_HostScope");
            await host.CallAsync(method, new { });

            await VerifyLeaseState(method, SingletonScope.Host, "TestValue", LeaseState.Available, LeaseStatus.Unlocked);

            host.Stop();
            host.Dispose();
        }
        public async Task SingletonFunction_StorageAccountOverride()
        {
            JobHost host = CreateTestJobHost(1);
            await host.StartAsync();

            MethodInfo method = typeof(TestJobs).GetMethod("SingletonJob_StorageAccountOverride");

            await host.CallAsync(method, new { message = "{}" });

            await host.StopAsync();

            host.Dispose();

            // make sure the lease blob was only created in the secondary account
            await VerifyLeaseDoesNotExistAsync(method, SingletonScope.Function, null);
            await VerifyLeaseState(method, SingletonScope.Function, null, LeaseState.Available, LeaseStatus.Unlocked, directory : _secondaryLockDirectory);
        }
Exemple #11
0
        /// <summary>
        /// Releases all the managed resources
        /// </summary>
        public void Dispose()
        {
            if (IsDisposed)
            {
                return;
            }

            if (_host.IsNotNull())
            {
                _host.Stop();
                _host.Dispose();
            }

            GC.SuppressFinalize(this);

            IsDisposed = true;
        }
        public async Task QueueFunction_SingletonListener()
        {
            JobHost host = CreateTestJobHost(1);
            await host.StartAsync();

            MethodInfo method = typeof(TestJobs).GetMethod("QueueFunction_SingletonListener");

            await VerifyLeaseState(method, SingletonScope.Function, "Listener", LeaseState.Leased, LeaseStatus.Locked);

            await host.CallAsync(method, new { message = "{}" });

            await host.StopAsync();

            host.Dispose();

            await VerifyLeaseState(method, SingletonScope.Function, "Listener", LeaseState.Available, LeaseStatus.Unlocked);
        }
        public async Task SingletonListener_SingletonFunction_ListenerSingletonOverride_InvocationsAreSerialized()
        {
            JobHost host = CreateTestJobHost(1);
            await host.StartAsync();

            MethodInfo singletonListenerAndFunctionMethod = typeof(TestJobs).GetMethod("SingletonTriggerJob_SingletonListener_ListenerSingletonOverride");

            await VerifyLeaseState(singletonListenerAndFunctionMethod, SingletonScope.Function, "TestScopeTestValue.Listener", LeaseState.Leased, LeaseStatus.Locked);

            await host.CallAsync(singletonListenerAndFunctionMethod, new { test = "Test" });

            await host.StopAsync();

            host.Dispose();

            await VerifyLeaseState(singletonListenerAndFunctionMethod, SingletonScope.Function, "TestScope", LeaseState.Available, LeaseStatus.Unlocked);
            await VerifyLeaseState(singletonListenerAndFunctionMethod, SingletonScope.Function, "TestScopeTestValue.Listener", LeaseState.Available, LeaseStatus.Unlocked);
        }
Exemple #14
0
        public async Task MultipleAccountTest()
        {
            try
            {
                TestTraceWriter trace = new TestTraceWriter(TraceLevel.Info);
                _serviceBusConfig = new ServiceBusConfiguration();
                _serviceBusConfig.MessagingProvider = new CustomMessagingProvider(_serviceBusConfig, trace);

                JobHostConfiguration config = new JobHostConfiguration()
                {
                    NameResolver = _nameResolver,
                    TypeLocator  = new FakeTypeLocator(typeof(ServiceBusTestJobs))
                };
                config.Tracing.Tracers.Add(trace);
                config.UseServiceBus(_serviceBusConfig);
                JobHost host = new JobHost(config);

                string queueName      = ResolveName(StartQueueName);
                string queuePrefix    = queueName.Replace("-queue-start", "");
                string firstTopicName = string.Format("{0}-topic/Subscriptions/{0}-queue-topic-1", queuePrefix);

                WriteQueueMessage(_secondaryNamespaceManager, _secondaryConnectionString, queueName, "Test");

                _topicSubscriptionCalled1 = new ManualResetEvent(initialState: false);

                await host.StartAsync();

                _topicSubscriptionCalled1.WaitOne(SBTimeout);

                // ensure all logs have had a chance to flush
                await Task.Delay(3000);

                // Wait for the host to terminate
                await host.StopAsync();

                host.Dispose();

                Assert.Equal("Test-topic-1", _resultMessage1);
            }
            finally
            {
                Cleanup();
            }
        }
        public async Task SingletonTriggerFunction_MultipleConcurrentInvocations_InvocationsAreSerialized()
        {
            JobHost host = CreateTestJobHost(1);

            host.Start();

            // trigger a bunch of parallel invocations
            int            numMessages = 20;
            List <Task>    invokeTasks = new List <Task>();
            JsonSerializer serializer  = new JsonSerializer();

            for (int i = 0; i < numMessages; i++)
            {
                int zone = _rand.Next(3) + 1;

                JObject workItem = new JObject
                {
                    { "ID", i + 1 },
                    { "Region", "Central" },
                    { "Zone", zone },
                    { "Category", 3 },
                    { "Description", "Test Work Item " + i }
                };
                await host.CallAsync(typeof(TestJobs).GetMethod("EnqueueQueue2TestMessage"), new { message = workItem.ToString() });
            }

            // wait for all the messages to be processed by the job
            await TestHelpers.Await(() =>
            {
                return((TestJobs.Queue2MessageCount == numMessages &&
                        TestJobs.JobInvocations.Select(p => p.Value).Sum() == numMessages) || TestJobs.FailureDetected);
            }, pollingInterval : 500);

            Assert.False(TestJobs.FailureDetected);
            Assert.Equal(numMessages, TestJobs.JobInvocations[1]);

            await VerifyLeaseState(typeof(TestJobs).GetMethod("SingletonTriggerJob"), SingletonScope.Function, "Central/1", LeaseState.Available, LeaseStatus.Unlocked);
            await VerifyLeaseState(typeof(TestJobs).GetMethod("SingletonTriggerJob"), SingletonScope.Function, "Central/2", LeaseState.Available, LeaseStatus.Unlocked);
            await VerifyLeaseState(typeof(TestJobs).GetMethod("SingletonTriggerJob"), SingletonScope.Function, "Central/3", LeaseState.Available, LeaseStatus.Unlocked);

            host.Stop();
            host.Dispose();
        }
Exemple #16
0
        public void IndexingExceptions_CanBeHandledByTraceWriter()
        {
            JobHostConfiguration config      = new JobHostConfiguration();
            TestTraceWriter      traceWriter = new TestTraceWriter(TraceLevel.Verbose);

            config.Tracing.Tracers.Add(traceWriter);
            config.TypeLocator = new FakeTypeLocator(typeof(BindingErrorsProgram));

            FunctionErrorTraceWriter errorTraceWriter = new FunctionErrorTraceWriter(TraceLevel.Error);

            config.Tracing.Tracers.Add(errorTraceWriter);

            JobHost host = new JobHost(config);

            host.Start();

            // verify the handled binding error
            FunctionIndexingException fex = errorTraceWriter.Errors.SingleOrDefault() as FunctionIndexingException;

            Assert.True(fex.Handled);
            Assert.Equal("BindingErrorsProgram.Invalid", fex.MethodName);

            // verify that the binding error was logged
            Assert.Equal(3, traceWriter.Traces.Count);
            TraceEvent traceEvent = traceWriter.Traces[0];

            Assert.Equal("Error indexing method 'BindingErrorsProgram.Invalid'", traceEvent.Message);
            Assert.Same(fex, traceEvent.Exception);
            Assert.Equal("Invalid container name: invalid$=+1", traceEvent.Exception.InnerException.Message);

            // verify that the valid function was still indexed
            traceEvent = traceWriter.Traces[1];
            Assert.True(traceEvent.Message.Contains("Found the following functions"));
            Assert.True(traceEvent.Message.Contains("BindingErrorsProgram.Valid"));

            // verify that the job host was started successfully
            traceEvent = traceWriter.Traces[2];
            Assert.Equal("Job host started", traceEvent.Message);

            host.Stop();
            host.Dispose();
        }
Exemple #17
0
        /// <inheritdoc />
        public async Task OnTimeoutExceptionAsync(ExceptionDispatchInfo exceptionInfo, TimeSpan timeoutGracePeriod)
        {
            try
            {
                // It's possible to have deadlocks while stopping. Make our best effort to stop
                // before disposing. We'll give it three seconds to stop before moving on.
                Task stopTask  = _host.StopAsync();
                Task delayTask = Task.Delay(TimeSpan.FromSeconds(3));

                await Task.WhenAny(stopTask, delayTask);
            }
            finally
            {
                _host.Dispose();
            }

            await Task.Delay(timeoutGracePeriod);

            await this.OnUnhandledExceptionAsync(exceptionInfo);
        }
Exemple #18
0
        public async Task MultipleAccountTest()
        {
            try
            {
                _serviceBusConfig = new ServiceBusConfiguration();
                _serviceBusConfig.MessagingProvider = new CustomMessagingProvider(_serviceBusConfig, null);

                JobHostConfiguration config = new JobHostConfiguration()
                {
                    NameResolver = _nameResolver,
                    TypeLocator  = new FakeTypeLocator(typeof(ServiceBusTestJobs))
                };
                config.UseServiceBus(_serviceBusConfig);
                JobHost host = new JobHost(config);

                await WriteQueueMessage(_secondaryConnectionString, FirstQueueName, "Test");

                _topicSubscriptionCalled1 = new ManualResetEvent(initialState: false);

                await host.StartAsync();

                _topicSubscriptionCalled1.WaitOne(SBTimeout);

                // ensure all logs have had a chance to flush
                await Task.Delay(3000);

                // Wait for the host to terminate
                await host.StopAsync();

                host.Dispose();

                Assert.Equal("Test-topic-1", _resultMessage1);
            }
            finally
            {
                await Cleanup();
            }
        }
        public async Task SingletonListener_MultipleHosts_OnlyOneHostRunsListener()
        {
            // create and start multiple hosts concurrently
            int            numHosts = 3;
            List <JobHost> hosts    = new List <JobHost>();

            Task[] tasks = new Task[numHosts];
            for (int i = 0; i < numHosts; i++)
            {
                JobHost host = CreateTestJobHost(i);
                hosts.Add(host);
                tasks[i] = host.StartAsync();
            }
            await Task.WhenAll(tasks);

            // verify that only 2 listeners were started (one for each of the singleton functions)
            Assert.Equal(3, TestTriggerAttributeBindingProvider.TestTriggerBinding.TestTriggerListener.StartCount);

            MethodInfo singletonListenerMethod = typeof(TestJobs).GetMethod("TriggerJob_SingletonListener");

            await VerifyLeaseState(singletonListenerMethod, SingletonScope.Function, "Listener", LeaseState.Leased, LeaseStatus.Locked);

            MethodInfo singletonListenerAndFunctionMethod = typeof(TestJobs).GetMethod("SingletonTriggerJob_SingletonListener");

            await VerifyLeaseState(singletonListenerAndFunctionMethod, SingletonScope.Function, "Listener", LeaseState.Leased, LeaseStatus.Locked);

            // stop all the hosts
            foreach (JobHost host in hosts)
            {
                await host.StopAsync();

                host.Dispose();
            }

            await VerifyLeaseState(singletonListenerMethod, SingletonScope.Function, "Listener", LeaseState.Available, LeaseStatus.Unlocked);
            await VerifyLeaseState(singletonListenerAndFunctionMethod, SingletonScope.Function, "Listener", LeaseState.Available, LeaseStatus.Unlocked);
        }
Exemple #20
0
        public async Task ServiceBusEndToEnd_CreatesEntities()
        {
            JobHost host      = null;
            var     startName = ResolveName(StartQueueName);
            var     topicName = ResolveName(TopicName);
            var     queueName = ResolveName(QueueNamePrefix);

            try
            {
                host = CreateHost(typeof(ServiceBusTestJobs_EntityCreation));
                await host.StartAsync();

                CreateStartMessage(_serviceBusConfig.ConnectionString, startName);
                CreateStartMessage(_serviceBusConfig.ConnectionString, startName + '1');
                CreateStartMessage(_serviceBusConfig.ConnectionString, startName + '2');

                await TestHelpers.Await(() =>
                {
                    return(_namespaceManager.TopicExists(topicName) &&
                           _namespaceManager.QueueExists(queueName + '1') &&
                           _namespaceManager.QueueExists(queueName + '2'));
                }, 30000);

                Assert.Throws <MessagingException>(() => _namespaceManager.QueueExists(topicName));
                Assert.Throws <MessagingException>(() => _namespaceManager.TopicExists(queueName + '1'));
                Assert.Throws <MessagingException>(() => _namespaceManager.TopicExists(queueName + '2'));
            }
            finally
            {
                host?.StopAsync();
                host?.Dispose();
                Cleanup();
                CleanupQueue(startName + '1');
                CleanupQueue(startName + '2');
                CleanupQueue(queueName + '2');
            }
        }
        public async Task MultipleAccountTest()
        {
            try
            {
                TestTraceWriter trace = new TestTraceWriter(TraceLevel.Info);
                _serviceBusConfig = new ServiceBusConfiguration();
                _serviceBusConfig.MessagingProvider = new CustomMessagingProvider(_serviceBusConfig, trace);

                JobHostConfiguration config = new JobHostConfiguration()
                {
                    NameResolver = _nameResolver,
                    TypeLocator = new FakeTypeLocator(typeof(ServiceBusTestJobs))
                };
                config.Tracing.Tracers.Add(trace);
                config.UseServiceBus(_serviceBusConfig);
                JobHost host = new JobHost(config);

                string queueName = ResolveName(StartQueueName);
                string queuePrefix = queueName.Replace("-queue-start", "");
                string firstTopicName = string.Format("{0}-topic/Subscriptions/{0}-queue-topic-1", queuePrefix);

                WriteQueueMessage(_secondaryNamespaceManager, _secondaryConnectionString, queueName, "Test");

                _topicSubscriptionCalled1 = new ManualResetEvent(initialState: false);

                await host.StartAsync();

                _topicSubscriptionCalled1.WaitOne(SBTimeout);

                // ensure all logs have had a chance to flush
                await Task.Delay(3000);

                // Wait for the host to terminate
                await host.StopAsync();
                host.Dispose();

                Assert.Equal("Test-topic-1", _resultMessage1);
            }
            finally
            {
                Cleanup();
            }
        }
Exemple #22
0
        private async Task ServiceBusEndToEndInternal(Type jobContainerType, JobHost host = null, bool verifyLogs = true)
        {
            if (host == null)
            {
                host = CreateHost(jobContainerType);
            }

            await WriteQueueMessage(_serviceBusConfig.ConnectionString, FirstQueueName, "E2E");

            _topicSubscriptionCalled1 = new ManualResetEvent(initialState: false);
            _topicSubscriptionCalled2 = new ManualResetEvent(initialState: false);

            await host.StartAsync();

            _topicSubscriptionCalled1.WaitOne(SBTimeout);
            _topicSubscriptionCalled2.WaitOne(SBTimeout);

            // ensure all logs have had a chance to flush
            await Task.Delay(3000);

            // Wait for the host to terminate
            await host.StopAsync();

            host.Dispose();

            Assert.Equal("E2E-SBQueue2SBQueue-SBQueue2SBTopic-topic-1", _resultMessage1);
            Assert.Equal("E2E-SBQueue2SBQueue-SBQueue2SBTopic-topic-2", _resultMessage2);

            if (verifyLogs)
            {
                IEnumerable <LogMessage> consoleOutput = _loggerProvider.GetAllLogMessages();

                Assert.DoesNotContain(consoleOutput, p => p.Level == LogLevel.Error);

                string[] consoleOutputLines = consoleOutput
                                              .Where(p => p.FormattedMessage != null)
                                              .SelectMany(p => p.FormattedMessage.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries))
                                              .OrderBy(p => p)
                                              .ToArray();

                string[] expectedOutputLines = new string[]
                {
                    "Found the following functions:",
                    $"{jobContainerType.FullName}.SBQueue2SBQueue",
                    $"{jobContainerType.FullName}.MultipleAccounts",
                    $"{jobContainerType.FullName}.SBQueue2SBTopic",
                    $"{jobContainerType.FullName}.SBTopicListener1",
                    $"{jobContainerType.FullName}.SBTopicListener2",
                    $"{jobContainerType.FullName}.ServiceBusBinderTest",
                    "Job host started",
                    $"Executing '{jobContainerType.Name}.SBQueue2SBQueue' (Reason='New ServiceBus message detected on '{FirstQueueName}'.', Id=",
                    $"Executed '{jobContainerType.Name}.SBQueue2SBQueue' (Succeeded, Id=",
                    $"Executing '{jobContainerType.Name}.SBQueue2SBTopic' (Reason='New ServiceBus message detected on '{SecondQueueName}'.', Id=",
                    $"Executed '{jobContainerType.Name}.SBQueue2SBTopic' (Succeeded, Id=",
                    $"Executing '{jobContainerType.Name}.SBTopicListener1' (Reason='New ServiceBus message detected on '{EntityNameHelper.FormatSubscriptionPath(TopicName, TopicSubscriptionName1)}'.', Id=",
                    $"Executed '{jobContainerType.Name}.SBTopicListener1' (Succeeded, Id=",
                    $"Executing '{jobContainerType.Name}.SBTopicListener2' (Reason='New ServiceBus message detected on '{EntityNameHelper.FormatSubscriptionPath(TopicName, TopicSubscriptionName2)}'.', Id=",
                    $"Executed '{jobContainerType.Name}.SBTopicListener2' (Succeeded, Id=",
                    "Job host stopped"
                }.OrderBy(p => p).ToArray();

                Action <string>[] inspectors = expectedOutputLines.Select <string, Action <string> >(p => (string m) => m.StartsWith(p)).ToArray();
                Assert.Collection(consoleOutputLines, inspectors);
            }
        }
        private async Task ServiceBusEndToEndInternal(Type jobContainerType, JobHost host = null, bool verifyLogs = true)
        {
            StringWriter consoleOutput = null;
            TextWriter   hold          = null;

            if (verifyLogs)
            {
                consoleOutput = new StringWriter();
                hold          = Console.Out;
                Console.SetOut(consoleOutput);
            }

            if (host == null)
            {
                host = CreateHost(jobContainerType);
            }

            await WriteQueueMessage(_serviceBusConfig.ConnectionString, FirstQueueName, "E2E");

            _topicSubscriptionCalled1 = new ManualResetEvent(initialState: false);
            _topicSubscriptionCalled2 = new ManualResetEvent(initialState: false);

            await host.StartAsync();

            _topicSubscriptionCalled1.WaitOne(SBTimeout);
            _topicSubscriptionCalled2.WaitOne(SBTimeout);

            // ensure all logs have had a chance to flush
            await Task.Delay(3000);

            // Wait for the host to terminate
            await host.StopAsync();

            host.Dispose();

            Assert.Equal("E2E-SBQueue2SBQueue-SBQueue2SBTopic-topic-1", _resultMessage1);
            Assert.Equal("E2E-SBQueue2SBQueue-SBQueue2SBTopic-topic-2", _resultMessage2);

            if (verifyLogs)
            {
                Console.SetOut(hold);

                string[] consoleOutputLines  = consoleOutput.ToString().Trim().Split(new string[] { Environment.NewLine }, StringSplitOptions.None).OrderBy(p => p).ToArray();
                string[] expectedOutputLines = new string[]
                {
                    "Found the following functions:",
                    string.Format("{0}.SBQueue2SBQueue", jobContainerType.FullName),
                    string.Format("{0}.MultipleAccounts", jobContainerType.FullName),
                    string.Format("{0}.SBQueue2SBTopic", jobContainerType.FullName),
                    string.Format("{0}.SBTopicListener1", jobContainerType.FullName),
                    string.Format("{0}.SBTopicListener2", jobContainerType.FullName),
                    "Job host started",
                    string.Format("Executing '{0}.SBQueue2SBQueue' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, FirstQueueName),
                    string.Format("Executed '{0}.SBQueue2SBQueue' (Succeeded, Id=", jobContainerType.Name),
                    string.Format("Executing '{0}.SBQueue2SBTopic' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, SecondQueueName),
                    string.Format("Executed '{0}.SBQueue2SBTopic' (Succeeded, Id=", jobContainerType.Name),
                    string.Format("Executing '{0}.SBTopicListener1' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, EntityNameHelper.FormatSubscriptionPath(TopicName, TopicSubscriptionName1)),
                    string.Format("Executed '{0}.SBTopicListener1' (Succeeded, Id=", jobContainerType.Name),
                    string.Format("Executing '{0}.SBTopicListener2' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, EntityNameHelper.FormatSubscriptionPath(TopicName, TopicSubscriptionName2)),
                    string.Format("Executed '{0}.SBTopicListener2' (Succeeded, Id=", jobContainerType.Name),
                    "Job host stopped"
                }.OrderBy(p => p).ToArray();

                bool hasError = consoleOutputLines.Any(p => p.Contains("Function had errors"));
                if (!hasError)
                {
                    for (int i = 0; i < expectedOutputLines.Length; i++)
                    {
                        Assert.StartsWith(expectedOutputLines[i], consoleOutputLines[i]);
                    }
                }
            }
        }
        public void IndexingExceptions_CanBeHandledByTraceWriter()
        {
            JobHostConfiguration config = new JobHostConfiguration();
            TestTraceWriter traceWriter = new TestTraceWriter(TraceLevel.Verbose);
            config.Tracing.Tracers.Add(traceWriter);
            config.TypeLocator = new FakeTypeLocator(typeof(BindingErrorsProgram));

            FunctionErrorTraceWriter errorTraceWriter = new FunctionErrorTraceWriter(TraceLevel.Error);
            config.Tracing.Tracers.Add(errorTraceWriter);

            JobHost host = new JobHost(config);
            host.Start();

            // verify the handled binding error
            FunctionIndexingException fex = errorTraceWriter.Errors.SingleOrDefault() as FunctionIndexingException;
            Assert.True(fex.Handled);
            Assert.Equal("BindingErrorsProgram.Invalid", fex.MethodName);

            // verify that the binding error was logged
            Assert.Equal(5, traceWriter.Traces.Count);
            TraceEvent traceEvent = traceWriter.Traces[0];
            Assert.Equal("Error indexing method 'BindingErrorsProgram.Invalid'", traceEvent.Message);
            Assert.Same(fex, traceEvent.Exception);
            Assert.Equal("Invalid container name: invalid$=+1", traceEvent.Exception.InnerException.Message);

            // verify that the valid function was still indexed
            traceEvent = traceWriter.Traces[1];
            Assert.True(traceEvent.Message.Contains("Found the following functions"));
            Assert.True(traceEvent.Message.Contains("BindingErrorsProgram.Valid"));

            // verify that the job host was started successfully
            traceEvent = traceWriter.Traces[4];
            Assert.Equal("Job host started", traceEvent.Message);

            host.Stop();
            host.Dispose();
        }