private async Task <WebScriptHostManager> CreateAndStartWebScriptHostManager()
{
var functions = new Collection <string> {
"TimeoutToken"
};
ScriptHostConfiguration config = new ScriptHostConfiguration()
{
RootScriptPath = $@"TestScripts\CSharp",
FileLoggingMode = FileLoggingMode.Always,
Functions = functions,
FunctionTimeout = TimeSpan.FromSeconds(3)
};
var loggerProviderFactory = new TestLoggerProviderFactory(_loggerProvider);
var mockEventManager = new Mock <IScriptEventManager>();
var mockRouter = new Mock <IWebJobsRouter>();
var manager = new WebScriptHostManager(
config,
new TestSecretManagerFactory(),
mockEventManager.Object,
ScriptSettingsManager.Instance,
new WebHostSettings {
SecretsPath = _secretsDirectory.Path
},
mockRouter.Object,
NullLoggerFactory.Instance);
Task task = Task.Run(() => { manager.RunAndBlock(); });
await TestHelpers.Await(() => manager.State == ScriptHostState.Running, userMessageCallback : () => "Expected host to be running");
return(manager);
}
public async Task RunAndBlock_SelfHost_Succeeds()
{
var loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
ScriptHostConfiguration config = new ScriptHostConfiguration()
{
RootScriptPath = Environment.CurrentDirectory,
IsSelfHost = true
};
ScriptHostManager manager = null;
LogMessage[] logs = null;
using (manager = new ScriptHostManager(config, loggerProviderFactory: loggerProviderFactory))
{
var tIgnore = Task.Run(() => manager.RunAndBlock());
await TestHelpers.Await(() =>
{
logs = loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).ToArray();
return(manager.State == ScriptHostState.Error || logs.Any(p => p.FormattedMessage.Contains("Job host started")));
});
Assert.Equal(ScriptHostState.Running, manager.State);
Assert.Equal(0, logs.Count(p => p.Level == LogLevel.Error));
}
}
protected ScriptHostEndToEndTestFixture(string rootPath, string testId, ProxyClientExecutor proxyClient = null, bool startHost = true, ICollection <string> functions = null, string functionsWorkerLanguage = null)
{
if (!string.IsNullOrEmpty(functionsWorkerLanguage))
{
Environment.SetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName, functionsWorkerLanguage);
}
_settingsManager = ScriptSettingsManager.Instance;
FixtureId = testId;
string connectionString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
QueueClient = storageAccount.CreateCloudQueueClient();
BlobClient = storageAccount.CreateCloudBlobClient();
TableClient = storageAccount.CreateCloudTableClient();
CreateTestStorageEntities().Wait();
// ApiHubTestHelper.SetDefaultConnectionFactory();
ScriptHostConfiguration config = new ScriptHostConfiguration()
{
RootScriptPath = rootPath,
FileLoggingMode = FileLoggingMode.Always,
};
if (functions != null)
{
config.OnConfigurationApplied = c => c.Functions = functions;
}
RequestConfiguration = new HttpConfiguration();
EventManager = new ScriptEventManager();
ScriptHostEnvironmentMock = new Mock <IScriptHostEnvironment>();
LoggerProvider = new TestLoggerProvider();
ILoggerProviderFactory loggerProviderFactory = new TestLoggerProviderFactory(LoggerProvider);
// Reset the timer logs first, since one of the tests will
// be checking them
TestHelpers.ClearFunctionLogs("TimerTrigger");
TestHelpers.ClearFunctionLogs("ListenerStartupException");
InitializeConfig(config);
Func <string, FunctionDescriptor> funcLookup = (name) => this.Host.GetFunctionOrNull(name);
var fastLogger = new FunctionInstanceLogger(funcLookup, new MetricsLogger());
config.HostConfig.AddService <IAsyncCollector <FunctionInstanceLogEntry> >(fastLogger);
Host = new ScriptHost(ScriptHostEnvironmentMock.Object, EventManager, config, _settingsManager,
proxyClient: proxyClient, loggerProviderFactory: loggerProviderFactory);
Host.Initialize();
if (startHost)
{
Host.HostStarted += (s, e) => _hostStartedEvent.Set();
Host.Start();
_hostStartedEvent.Wait(TimeSpan.FromSeconds(30));
}
}
public InstanceManagerTests()
{
_httpClient = new HttpClient();
_loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(_loggerProvider);
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(_loggerProvider);
_settingsManager = new ScriptSettingsManager();
_instanceManager = new InstanceManager(_settingsManager, null, loggerFactory, _httpClient);
}
private async Task InitializeTestHost(string testDirName)
{
var httpConfig = new HttpConfiguration();
var testRootPath = Path.Combine(Path.GetTempPath(), testDirName);
await FileUtility.DeleteDirectoryAsync(testRootPath, true);
_loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(_loggerProvider);
_webHostSettings = new WebHostSettings
{
IsSelfHost = true,
LogPath = Path.Combine(testRootPath, "Logs"),
SecretsPath = Path.Combine(testRootPath, "Secrets"),
ScriptPath = Path.Combine(testRootPath, "WWWRoot")
};
if (_settingsManager.IsAppServiceEnvironment)
{
// if the test is mocking App Service environment, we need
// to also set the HOME variable
Environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHomePath, testRootPath);
}
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(_loggerProvider);
var webHostBuilder = Program.CreateWebHostBuilder()
.ConfigureServices(c =>
{
c.AddSingleton(_webHostSettings)
.AddSingleton <ILoggerProviderFactory>(loggerProviderFactory)
.AddSingleton <ILoggerFactory>(loggerFactory);
});
_httpServer = new TestServer(webHostBuilder);
_httpClient = _httpServer.CreateClient();
_httpClient.BaseAddress = new Uri("https://localhost/");
TestHelpers.WaitForWebHost(_httpClient);
var traces = _loggerProvider.GetAllLogMessages().ToArray();
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Starting Host (HostId=placeholder-host")));
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Host is in standby mode")));
var hostConfig = WebHostResolver.CreateScriptHostConfiguration(_webHostSettings, true);
_expectedHostId = hostConfig.HostConfig.HostId;
}
public async Task RunAndBlock_ParseError_LogsError()
{
TestLoggerProvider loggerProvider = new TestLoggerProvider();
TestLoggerProviderFactory factory = new TestLoggerProviderFactory(loggerProvider, includeDefaultLoggerProviders: false);
string rootPath = Path.Combine(Environment.CurrentDirectory, "ScriptHostTests");
if (!Directory.Exists(rootPath))
{
Directory.CreateDirectory(rootPath);
}
var configPath = Path.Combine(rootPath, "host.json");
File.WriteAllText(configPath, @"{<unparseable>}");
var config = new ScriptHostConfiguration()
{
RootScriptPath = rootPath
};
config.HostConfig.HostId = ID;
var scriptHostFactory = new TestScriptHostFactory();
var eventManagerMock = new Mock <IScriptEventManager>();
var hostManager = new ScriptHostManager(config, _settingsManager, scriptHostFactory, eventManagerMock.Object, loggerProviderFactory: factory);
Task taskIgnore = Task.Run(() => hostManager.RunAndBlock());
await TestHelpers.Await(() => hostManager.State == ScriptHostState.Error, 3000, 50);
Assert.Equal(ScriptHostState.Error, hostManager.State);
hostManager.Stop();
var ex = hostManager.LastError;
Assert.True(ex is FormatException);
var expectedMessage = $"Unable to parse host configuration file '{configPath}'.";
Assert.Equal(expectedMessage, ex.Message);
var logger = loggerProvider.CreatedLoggers.Last();
var logMessage = logger.GetLogMessages()[0];
Assert.StartsWith("A ScriptHost error has occurred", logMessage.FormattedMessage);
Assert.Equal(expectedMessage, logMessage.Exception.Message);
}
public async Task RunAndBlock_HostJsonValueError_LogsError()
{
// Try to load valid host.json file that has an out-of-range value.
// Ensure that it's logged to ILogger
string rootPath = Path.Combine(Environment.CurrentDirectory, @"TestScripts\OutOfRange");
ScriptHostConfiguration config = new ScriptHostConfiguration()
{
RootScriptPath = rootPath
};
TestLoggerProvider provider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(provider, includeDefaultLoggerProviders: false);
var factoryMock = new Mock <IScriptHostFactory>();
var scriptHostFactory = new TestScriptHostFactory();
var eventManagerMock = new Mock <IScriptEventManager>();
var hostManager = new ScriptHostManager(config, _settingsManager, scriptHostFactory, eventManagerMock.Object, loggerProviderFactory: loggerProviderFactory);
Task taskIgnore = Task.Run(() => hostManager.RunAndBlock());
await TestHelpers.Await(() => hostManager.State == ScriptHostState.Error, 3000, 50);
Assert.Equal(ScriptHostState.Error, hostManager.State);
Assert.False(hostManager.CanInvoke());
hostManager.Stop();
var ex = hostManager.LastError;
Assert.True(ex is ArgumentOutOfRangeException);
string msg = "A ScriptHost error has occurred";
var startupLogger = provider.CreatedLoggers.Last();
var loggerMessage = startupLogger.GetLogMessages().First();
Assert.Equal(msg, loggerMessage.FormattedMessage);
Assert.Same(ex, loggerMessage.Exception);
}
public async Task StandbyMode_EndToEnd()
{
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
{ EnvironmentSettingNames.AzureWebsiteInstanceId, "87654639876900123453445678890144" },
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
var httpConfig = new HttpConfiguration();
var testRootPath = Path.Combine(Path.GetTempPath(), "StandbyModeTest");
await FileUtility.DeleteDirectoryAsync(testRootPath, true);
var loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
var webHostSettings = new WebHostSettings
{
IsSelfHost = true,
LogPath = Path.Combine(testRootPath, "Logs"),
SecretsPath = Path.Combine(testRootPath, "Secrets"),
ScriptPath = Path.Combine(testRootPath, "WWWRoot")
};
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(loggerProvider);
var webHostBuilder = Program.CreateWebHostBuilder()
.ConfigureServices(c => {
c.AddSingleton(webHostSettings)
.AddSingleton <ILoggerProviderFactory>(loggerProviderFactory)
.AddSingleton <ILoggerFactory>(loggerFactory);
});
var httpServer = new TestServer(webHostBuilder);
var httpClient = httpServer.CreateClient();
httpClient.BaseAddress = new Uri("https://localhost/");
TestHelpers.WaitForWebHost(httpClient);
var traces = loggerProvider.GetAllLogMessages().ToArray();
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Starting Host (HostId=placeholder-host")));
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Host is in standby mode")));
// issue warmup request and verify
var request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
var response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
string responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// issue warmup request with restart and verify
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup?restart=1");
response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// Now specialize the host
ScriptSettingsManager.Instance.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
ScriptSettingsManager.Instance.SetSetting(EnvironmentSettingNames.AzureWebsiteContainerReady, "1");
Assert.False(WebScriptHostManager.InStandbyMode);
Assert.True(ScriptSettingsManager.Instance.ContainerReady);
// give time for the specialization to happen
string[] logLines = null;
await TestHelpers.Await(() =>
{
// wait for the trace indicating that the host has been specialized
logLines = loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
return(logLines.Contains("Generating 0 job function(s)"));
}, userMessageCallback : () => string.Join(Environment.NewLine, loggerProvider.GetAllLogMessages().Select(p => $"[{p.Timestamp.ToString("HH:mm:ss.fff")}] {p.FormattedMessage}")));
httpServer.Dispose();
httpClient.Dispose();
await Task.Delay(2000);
var hostConfig = WebHostResolver.CreateScriptHostConfiguration(webHostSettings, true);
var expectedHostId = hostConfig.HostConfig.HostId;
// verify the rest of the expected logs
string text = string.Join(Environment.NewLine, logLines);
Assert.True(logLines.Count(p => p.Contains("Stopping Host")) >= 1);
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
Assert.Equal(2, logLines.Count(p => p.Contains("Starting Host (HostId=placeholder-host")));
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Starting Host (HostId={expectedHostId}")));
Assert.Contains("Generating 0 job function(s)", logLines);
WebScriptHostManager.ResetStandbyMode();
}
}
public async Task StandbyMode_EndToEnd_LinuxContainer()
{
byte[] bytes = TestHelpers.GenerateKeyBytes();
var encryptionKey = Convert.ToBase64String(bytes);
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.ContainerName, "TestContainer" },
{ EnvironmentSettingNames.ContainerEncryptionKey, encryptionKey },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
var httpConfig = new HttpConfiguration();
var testRootPath = Path.Combine(Path.GetTempPath(), "StandbyModeTest_Linux");
await FileUtility.DeleteDirectoryAsync(testRootPath, true);
var loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
var webHostSettings = new WebHostSettings
{
IsSelfHost = true,
LogPath = Path.Combine(testRootPath, "Logs"),
SecretsPath = Path.Combine(testRootPath, "Secrets"),
ScriptPath = Path.Combine(testRootPath, "WWWRoot")
};
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(loggerProvider);
var webHostBuilder = Program.CreateWebHostBuilder()
.ConfigureServices(c =>
{
c.AddSingleton(webHostSettings)
.AddSingleton <ILoggerProviderFactory>(loggerProviderFactory)
.AddSingleton <ILoggerFactory>(loggerFactory);
});
var httpServer = new TestServer(webHostBuilder);
var httpClient = httpServer.CreateClient();
httpClient.BaseAddress = new Uri("https://localhost/");
TestHelpers.WaitForWebHost(httpClient);
var traces = loggerProvider.GetAllLogMessages().ToArray();
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Starting Host (HostId=placeholder-host")));
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Host is in standby mode")));
// issue warmup request and verify
var request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
var response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
string responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// issue warmup request with restart and verify
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup?restart=1");
response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// Now specialize the host by invoking assign
var secretManager = httpServer.Host.Services.GetService <ISecretManager>();
var masterKey = (await secretManager.GetHostSecretsAsync()).MasterKey;
string uri = "admin/instance/assign";
request = new HttpRequestMessage(HttpMethod.Post, uri);
var environment = new Dictionary <string, string>();
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestSite",
Environment = environment
};
var encryptedAssignmentContext = EncryptedHostAssignmentContext.Create(assignmentContext, encryptionKey);
string json = JsonConvert.SerializeObject(encryptedAssignmentContext);
request.Content = new StringContent(json, Encoding.UTF8, "application/json");
request.Headers.Add(AuthenticationLevelHandler.FunctionsKeyHeaderName, masterKey);
response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
// give time for the specialization to happen
string[] logLines = null;
await TestHelpers.Await(() =>
{
// wait for the trace indicating that the host has been specialized
logLines = loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
return(logLines.Contains("Generating 0 job function(s)"));
}, userMessageCallback : () => string.Join(Environment.NewLine, loggerProvider.GetAllLogMessages().Select(p => $"[{p.Timestamp.ToString("HH:mm:ss.fff")}] {p.FormattedMessage}")));
httpServer.Dispose();
httpClient.Dispose();
await Task.Delay(2000);
var hostConfig = WebHostResolver.CreateScriptHostConfiguration(webHostSettings, true);
var expectedHostId = hostConfig.HostConfig.HostId;
// verify the rest of the expected logs
string text = string.Join(Environment.NewLine, logLines);
Assert.True(logLines.Count(p => p.Contains("Stopping Host")) >= 1);
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
Assert.Equal(2, logLines.Count(p => p.Contains("Starting Host (HostId=placeholder-host")));
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Starting Host (HostId={expectedHostId}")));
Assert.Contains("Generating 0 job function(s)", logLines);
WebScriptHostManager.ResetStandbyMode();
}
}
public async Task HostHealthMonitor_TriggersShutdown_WhenHostUnhealthy()
{
string functionDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Functions", Guid.NewGuid().ToString());
Directory.CreateDirectory(functionDir);
string logDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Logs", Guid.NewGuid().ToString());
JObject hostConfig = new JObject
{
{ "id", "123456" }
};
File.WriteAllText(Path.Combine(functionDir, ScriptConstants.HostMetadataFileName), hostConfig.ToString());
var config = new ScriptHostConfiguration
{
RootScriptPath = functionDir,
RootLogPath = logDir,
FileLoggingMode = FileLoggingMode.Always,
};
// configure the monitor so it will fail within a couple seconds
config.HostHealthMonitor.HealthCheckInterval = TimeSpan.FromMilliseconds(100);
config.HostHealthMonitor.HealthCheckWindow = TimeSpan.FromSeconds(1);
config.HostHealthMonitor.HealthCheckThreshold = 5;
var environmentMock = new Mock <IScriptHostEnvironment>(MockBehavior.Strict);
environmentMock.Setup(p => p.Shutdown());
var mockSettings = new Mock <ScriptSettingsManager>();
mockSettings.Setup(p => p.IsAzureEnvironment).Returns(true);
var eventManagerMock = new Mock <IScriptEventManager>();
var hostHealthConfig = new HostHealthMonitorConfiguration();
var mockHostPerformanceManager = new Mock <HostPerformanceManager>(mockSettings.Object, hostHealthConfig);
bool underHighLoad = false;
mockHostPerformanceManager.Setup(p => p.IsUnderHighLoad(It.IsAny <Collection <string> >(), It.IsAny <ILogger>()))
.Callback <Collection <string>, ILogger>((c, l) =>
{
c.Add("Connections");
})
.Returns(() => underHighLoad);
var loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
var hostManager = new ScriptHostManager(config, mockSettings.Object, new ScriptHostFactory(), eventManagerMock.Object, environmentMock.Object, loggerProviderFactory, mockHostPerformanceManager.Object);
Assert.True(hostManager.ShouldMonitorHostHealth);
Task runTask = Task.Run(() => hostManager.RunAndBlock());
await TestHelpers.Await(() => hostManager.State == ScriptHostState.Running);
// now that host is running make host unhealthy and wait
// for host shutdown
underHighLoad = true;
await TestHelpers.Await(() => hostManager.State == ScriptHostState.Error);
Assert.Equal(ScriptHostState.Error, hostManager.State);
environmentMock.Verify(p => p.Shutdown(), Times.Once);
// we expect a few restart iterations
var thresholdErrors = loggerProvider.GetAllLogMessages().Where(p => p.Exception is InvalidOperationException && p.Exception.Message == "Host thresholds exceeded: [Connections]. For more information, see https://aka.ms/functions-thresholds.");
Assert.True(thresholdErrors.Count() > 1);
var log = loggerProvider.GetAllLogMessages().Last();
Assert.True(loggerProvider.GetAllLogMessages().Count(p => p.FormattedMessage == "Host is unhealthy. Initiating a restart." && p.Level == LogLevel.Error) > 0);
Assert.Equal("Host unhealthy count exceeds the threshold of 5 for time window 00:00:01. Initiating shutdown.", log.FormattedMessage);
Assert.Equal(LogLevel.Error, log.Level);
}
