public void GetCounters_ReturnsExpectedResult() { var mockSettings = new Mock <ScriptSettingsManager>(MockBehavior.Strict, null); string value = string.Empty; var logger = new TestLogger("Test"); mockSettings.Setup(p => p.GetSetting(EnvironmentSettingNames.AzureWebsiteAppCountersName)).Returns(() => value); var healthMonitorConfig = new HostHealthMonitorConfiguration(); var performanceManager = new HostPerformanceManager(mockSettings.Object, healthMonitorConfig); value = "{\"userTime\": 30000000,\"kernelTime\": 16562500,\"pageFaults\": 131522,\"processes\": 1,\"processLimit\": 32,\"threads\": 32,\"threadLimit\": 512,\"connections\": 4,\"connectionLimit\": 300,\"sections\": 3,\"sectionLimit\": 256,\"namedPipes\": 0,\"namedPipeLimit\": 128,\"readIoOperations\": 675,\"writeIoOperations\": 18,\"otherIoOperations\": 9721,\"readIoBytes\": 72585119,\"writeIoBytes\": 5446,\"otherIoBytes\": 393926,\"privateBytes\": 33759232,\"handles\": 987,\"contextSwitches\": 15535,\"remoteOpens\": 250}"; var counters = performanceManager.GetPerformanceCounters(logger); Assert.Equal(counters.PageFaults, 131522); value = "{\"userTime\": 30000000,\"kernelTime\": 16562500,\"pageFaults\": 131522,\"processes\": 1,\"processLimit\": 32,\"threads\": 32,\"threadLimit\": 512,\"connections\": 4,\"connectionLimit\": 300,\"sections\": 3,\"sectionLimit\": 256,\"namedPipes\": 0,\"namedPipeLimit\": 128,\"readIoOperations\": 675,\"writeIoOperations\": 18,\"otherIoOperations\": 9721,\"readIoBytes\": 72585119,\"writeIoBytes\": 5446,\"otherIoBytes\": 393926,\"privateBytes\": 33759232,\"handles\": 987,\"contextSwitches\": 15535,\"remoteOpens\": 250}猅"; counters = performanceManager.GetPerformanceCounters(logger); Assert.Equal(counters.PageFaults, 131522); value = "{}"; counters = performanceManager.GetPerformanceCounters(logger); Assert.Equal(counters.PageFaults, 0); value = "this is not json"; counters = performanceManager.GetPerformanceCounters(logger); Assert.Null(counters); var error = logger.GetLogMessages().Last(); Assert.Equal("Failed to deserialize application performance counters. JSON Content: \"this is not json\"", error.FormattedMessage); }
public void IsHostHealthy_ReturnsExpectedResult() { var config = new ScriptHostConfiguration() { RootScriptPath = Environment.CurrentDirectory }; var mockSettings = new Mock <ScriptSettingsManager>(MockBehavior.Strict); var eventManager = new Mock <IScriptEventManager>(); var hostMock = new Mock <ScriptHost>(new NullScriptHostEnvironment(), eventManager.Object, config, null, null, null); var factoryMock = new Mock <IScriptHostFactory>(); factoryMock.Setup(f => f.Create(It.IsAny <IScriptHostEnvironment>(), It.IsAny <IScriptEventManager>(), mockSettings.Object, It.IsAny <ScriptHostConfiguration>(), It.IsAny <ILoggerProviderFactory>())) .Returns(hostMock.Object); var hostHealthConfig = new HostHealthMonitorConfiguration(); var mockHostPerformanceManager = new Mock <HostPerformanceManager>(mockSettings.Object, hostHealthConfig); var target = new Mock <ScriptHostManager>(config, mockSettings.Object, factoryMock.Object, eventManager.Object, new NullScriptHostEnvironment(), null, mockHostPerformanceManager.Object); Collection <string> exceededCounters = new Collection <string>(); bool isUnderHighLoad = false; mockHostPerformanceManager.Setup(p => p.IsUnderHighLoad(It.IsAny <Collection <string> >(), It.IsAny <ILogger>())) .Callback <Collection <string>, ILogger>((c, t) => { foreach (var counter in exceededCounters) { c.Add(counter); } }) .Returns(() => isUnderHighLoad); bool isAzureEnvironment = false; mockSettings.Setup(p => p.IsAzureEnvironment).Returns(() => isAzureEnvironment); mockSettings.Setup(p => p.FileSystemIsReadOnly).Returns(false); config.HostHealthMonitor.Enabled = false; Assert.True(target.Object.IsHostHealthy()); config.HostHealthMonitor.Enabled = true; Assert.True(target.Object.IsHostHealthy()); isAzureEnvironment = true; Assert.True(target.Object.IsHostHealthy()); isUnderHighLoad = true; exceededCounters.Add("Foo"); exceededCounters.Add("Bar"); Assert.False(target.Object.IsHostHealthy()); var ex = Assert.Throws <InvalidOperationException>(() => target.Object.IsHostHealthy(true)); Assert.Equal("Host thresholds exceeded: [Foo, Bar]. For more information, see https://aka.ms/functions-thresholds.", ex.Message); }
public HostPerformanceManager(ScriptSettingsManager settingsManager, HostHealthMonitorConfiguration healthMonitorConfig) { if (settingsManager == null) { throw new ArgumentNullException(nameof(settingsManager)); } if (healthMonitorConfig == null) { throw new ArgumentNullException(nameof(healthMonitorConfig)); } _settingsManager = settingsManager; _healthMonitorConfig = healthMonitorConfig; }
public async Task HostHealthMonitor_TriggersShutdown_WhenHostUnhealthy() { string functionDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Functions", Guid.NewGuid().ToString()); Directory.CreateDirectory(functionDir); string logDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Logs", Guid.NewGuid().ToString()); JObject hostConfig = new JObject { { "id", "123456" } }; File.WriteAllText(Path.Combine(functionDir, ScriptConstants.HostMetadataFileName), hostConfig.ToString()); var config = new ScriptHostConfiguration { RootScriptPath = functionDir, RootLogPath = logDir, FileLoggingMode = FileLoggingMode.Always, }; // configure the monitor so it will fail within a couple seconds config.HostHealthMonitor.HealthCheckInterval = TimeSpan.FromMilliseconds(100); config.HostHealthMonitor.HealthCheckWindow = TimeSpan.FromSeconds(1); config.HostHealthMonitor.HealthCheckThreshold = 5; var environmentMock = new Mock <IScriptHostEnvironment>(MockBehavior.Strict); environmentMock.Setup(p => p.Shutdown()); var mockSettings = new Mock <ScriptSettingsManager>(); mockSettings.Setup(p => p.IsAzureEnvironment).Returns(true); var eventManagerMock = new Mock <IScriptEventManager>(); var hostHealthConfig = new HostHealthMonitorConfiguration(); var mockHostPerformanceManager = new Mock <HostPerformanceManager>(mockSettings.Object, hostHealthConfig); bool underHighLoad = false; mockHostPerformanceManager.Setup(p => p.IsUnderHighLoad(It.IsAny <Collection <string> >(), It.IsAny <ILogger>())) .Callback <Collection <string>, ILogger>((c, l) => { c.Add("Connections"); }) .Returns(() => underHighLoad); var loggerProvider = new TestLoggerProvider(); var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider); var hostManager = new ScriptHostManager(config, mockSettings.Object, new ScriptHostFactory(), eventManagerMock.Object, environmentMock.Object, loggerProviderFactory, mockHostPerformanceManager.Object); Assert.True(hostManager.ShouldMonitorHostHealth); Task runTask = Task.Run(() => hostManager.RunAndBlock()); await TestHelpers.Await(() => hostManager.State == ScriptHostState.Running); // now that host is running make host unhealthy and wait // for host shutdown underHighLoad = true; await TestHelpers.Await(() => hostManager.State == ScriptHostState.Error); Assert.Equal(ScriptHostState.Error, hostManager.State); environmentMock.Verify(p => p.Shutdown(), Times.Once); // we expect a few restart iterations var thresholdErrors = loggerProvider.GetAllLogMessages().Where(p => p.Exception is InvalidOperationException && p.Exception.Message == "Host thresholds exceeded: [Connections]. For more information, see https://aka.ms/functions-thresholds."); Assert.True(thresholdErrors.Count() > 1); var log = loggerProvider.GetAllLogMessages().Last(); Assert.True(loggerProvider.GetAllLogMessages().Count(p => p.FormattedMessage == "Host is unhealthy. Initiating a restart." && p.Level == LogLevel.Error) > 0); Assert.Equal("Host unhealthy count exceeds the threshold of 5 for time window 00:00:01. Initiating shutdown.", log.FormattedMessage); Assert.Equal(LogLevel.Error, log.Level); }