private static RpcFunctionInvocationDispatcher GetTestFunctionDispatcher(string maxProcessCountValue = null, bool addWebhostChannel = false, Mock <IWebHostRpcWorkerChannelManager> mockwebHostLanguageWorkerChannelManager = null, bool throwOnProcessStartUp = false) { var eventManager = new ScriptEventManager(); var metricsLogger = new Mock <IMetricsLogger>(); var mockApplicationLifetime = new Mock <IApplicationLifetime>(); var testEnv = new TestEnvironment(); if (!string.IsNullOrEmpty(maxProcessCountValue)) { testEnv.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerProcessCountSettingName, maxProcessCountValue); } var loggerFactory = MockNullLoggerFactory.CreateLoggerFactory(); var options = new ScriptJobHostOptions { RootLogPath = Path.GetTempPath() }; IOptions <ScriptJobHostOptions> scriptOptions = new OptionsManager <ScriptJobHostOptions>(new TestOptionsFactory <ScriptJobHostOptions>(options)); var workerConfigOptions = new LanguageWorkerOptions { WorkerConfigs = TestHelpers.GetTestWorkerConfigs() }; IRpcWorkerChannelFactory testLanguageWorkerChannelFactory = new TestRpcWorkerChannelFactory(eventManager, _testLogger, scriptOptions.Value.RootScriptPath, throwOnProcessStartUp); IWebHostRpcWorkerChannelManager testWebHostLanguageWorkerChannelManager = new TestRpcWorkerChannelManager(eventManager, _testLogger, scriptOptions.Value.RootScriptPath, testLanguageWorkerChannelFactory); IJobHostRpcWorkerChannelManager jobHostLanguageWorkerChannelManager = new JobHostRpcWorkerChannelManager(loggerFactory); if (addWebhostChannel) { testWebHostLanguageWorkerChannelManager.InitializeChannelAsync("java"); } if (mockwebHostLanguageWorkerChannelManager != null) { testWebHostLanguageWorkerChannelManager = mockwebHostLanguageWorkerChannelManager.Object; } var mockFunctionDispatcherLoadBalancer = new Mock <IRpcFunctionInvocationDispatcherLoadBalancer>(); _javaTestChannel = new TestRpcWorkerChannel(Guid.NewGuid().ToString(), "java", eventManager, _testLogger, false); var optionsMonitor = TestHelpers.CreateOptionsMonitor(workerConfigOptions); return(new RpcFunctionInvocationDispatcher(scriptOptions, metricsLogger.Object, testEnv, mockApplicationLifetime.Object, eventManager, loggerFactory, testLanguageWorkerChannelFactory, optionsMonitor, testWebHostLanguageWorkerChannelManager, jobHostLanguageWorkerChannelManager, new OptionsWrapper <ManagedDependencyOptions>(new ManagedDependencyOptions()), mockFunctionDispatcherLoadBalancer.Object)); }
public async Task FunctionDispatcher_Error_BeyondThreshold_BucketIsAtOne() { RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(); await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName)); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1); for (int i = 1; i < 10; ++i) { foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels()) { TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel; testWorkerChannel.RaiseWorkerErrorWithCustomTimestamp(DateTime.UtcNow.AddHours(i)); } } Assert.Equal(1, functionDispatcher.LanguageWorkerErrors.Count); }
public async Task FunctionDispatcher_Error_WithinThreshold_BucketFills() { RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(1); await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName)); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1); for (int i = 0; i < 3; ++i) { foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels()) { TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel; testWorkerChannel.RaiseWorkerError(); } } Assert.Equal(3, functionDispatcher.LanguageWorkerErrors.Count); }
public async Task FunctionDispatcher_Restart_ErroredChannels_ExceedsLimit() { int expectedProcessCount = 2; RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount); await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName)); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount); for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++) { foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels()) { TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel; testWorkerChannel.RaiseWorkerError(); } } Assert.Equal(0, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count()); }
public RpcInitializationServiceTests() { _mockLanguageWorkerChannelManager = new Mock <IWebHostRpcWorkerChannelManager>(); _loggerFactory = new LoggerFactory(); _logger = _loggerFactory.CreateLogger <RpcInitializationService>(); var applicationHostOptions = new ScriptApplicationHostOptions { IsSelfHost = true, ScriptPath = Path.GetTempPath() }; _optionsMonitor = TestHelpers.CreateOptionsMonitor(applicationHostOptions); IRpcWorkerChannel testLanguageWorkerChannel = new TestRpcWorkerChannel(Guid.NewGuid().ToString(), RpcWorkerConstants.NodeLanguageWorkerName); _mockLanguageWorkerChannelManager.Setup(m => m.InitializeChannelAsync(It.IsAny <string>())) .Returns(Task.FromResult <IRpcWorkerChannel>(testLanguageWorkerChannel)); }
public async Task FunctionDispatcher_Restart_ErroredChannels_And_DoesNot_Change_State() { int expectedProcessCount = 2; RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount); Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State); // Add worker await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName)); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount); TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault(); // Restart one channel testWorkerChannel.RaiseWorkerRestart(); Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount); Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State); }
public async Task FunctionDispatcher_Restart_ErroredChannels_Succeeds() { int expectedProcessCount = 2; RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount); await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName)); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount); int finalChannelCount = 0; for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++) { TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault(); if (functionDispatcher.LanguageWorkerErrors.Count < (expectedProcessCount * 3) - 1) { testWorkerChannel.RaiseWorkerError(); } finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount); } Assert.Equal(expectedProcessCount, finalChannelCount); }
public async Task FunctionDispatcher_Restart_ErroredChannels_And_Changes_State() { int expectedProcessCount = 1; RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString()); Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State); // Add worker await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName)); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount); TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault(); // Restart this channel testWorkerChannel.RaiseWorkerRestart(); await TestHelpers.Await(() => { return(functionDispatcher.State == FunctionInvocationDispatcherState.WorkerProcessRestarting); }, 3000); await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount); Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State); }
private static RpcFunctionInvocationDispatcher GetTestFunctionDispatcher(int maxProcessCountValue = 1, bool addWebhostChannel = false, Mock <IWebHostRpcWorkerChannelManager> mockwebHostLanguageWorkerChannelManager = null, bool throwOnProcessStartUp = false, TimeSpan?startupIntervals = null, string runtime = null, bool workerIndexing = false, bool placeholder = false) { var eventManager = new ScriptEventManager(); var metricsLogger = new Mock <IMetricsLogger>(); var mockApplicationLifetime = new Mock <IApplicationLifetime>(); var testEnv = new TestEnvironment(); TimeSpan intervals = startupIntervals ?? TimeSpan.FromMilliseconds(100); testEnv.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerProcessCountSettingName, maxProcessCountValue.ToString()); if (runtime != null) { testEnv.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, runtime); } if (workerIndexing) { testEnv.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing); } if (placeholder) { testEnv.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1"); } var options = new ScriptJobHostOptions { RootLogPath = Path.GetTempPath() }; IOptions <ScriptJobHostOptions> scriptOptions = new OptionsManager <ScriptJobHostOptions>(new TestOptionsFactory <ScriptJobHostOptions>(options)); var workerConfigOptions = new LanguageWorkerOptions { WorkerConfigs = TestHelpers.GetTestWorkerConfigs(processCountValue: maxProcessCountValue, processStartupInterval: intervals, processRestartInterval: intervals, processShutdownTimeout: TimeSpan.FromSeconds(1), workerIndexing: workerIndexing) }; IRpcWorkerChannelFactory testLanguageWorkerChannelFactory = new TestRpcWorkerChannelFactory(eventManager, _testLogger, scriptOptions.Value.RootScriptPath, throwOnProcessStartUp); IWebHostRpcWorkerChannelManager testWebHostLanguageWorkerChannelManager = new TestRpcWorkerChannelManager(eventManager, _testLogger, scriptOptions.Value.RootScriptPath, testLanguageWorkerChannelFactory); IJobHostRpcWorkerChannelManager jobHostLanguageWorkerChannelManager = new JobHostRpcWorkerChannelManager(_testLoggerFactory); if (addWebhostChannel) { testWebHostLanguageWorkerChannelManager.InitializeChannelAsync("java"); } if (mockwebHostLanguageWorkerChannelManager != null) { testWebHostLanguageWorkerChannelManager = mockwebHostLanguageWorkerChannelManager.Object; } var mockFunctionDispatcherLoadBalancer = new Mock <IRpcFunctionInvocationDispatcherLoadBalancer>(); _javaTestChannel = new TestRpcWorkerChannel(Guid.NewGuid().ToString(), "java", eventManager, _testLogger, false); var optionsMonitor = TestHelpers.CreateOptionsMonitor(workerConfigOptions); return(new RpcFunctionInvocationDispatcher(scriptOptions, metricsLogger.Object, testEnv, mockApplicationLifetime.Object, eventManager, _testLoggerFactory, testLanguageWorkerChannelFactory, optionsMonitor, testWebHostLanguageWorkerChannelManager, jobHostLanguageWorkerChannelManager, new OptionsWrapper <ManagedDependencyOptions>(new ManagedDependencyOptions()), mockFunctionDispatcherLoadBalancer.Object, Options.Create(new WorkerConcurrencyOptions()))); }