public void GetAdminToken_Fails_NotLinuxContainer()
{
// Arrange
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(It.Is <string>(k => k == EnvironmentSettingNames.ContainerName))).Returns <string>(v => v = null);
var key = TestHelpers.GenerateKeyBytes();
var stringKey = TestHelpers.GenerateKeyHexString(key);
using (new TestScopedEnvironmentVariable(EnvironmentSettingNames.WebSiteAuthEncryptionKey, stringKey))
{
// Act
ObjectResult result = (ObjectResult)_hostController.GetAdminToken();
HttpStatusCode resultStatus = (HttpStatusCode)result.StatusCode;
// Assert
Assert.Equal(HttpStatusCode.BadRequest, resultStatus);
}
}
public void GetAdminToken_Succeeds()
{
// Arrange
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(It.Is <string>(k => k == EnvironmentSettingNames.ContainerName))).Returns <string>(v => v = "ContainerName");
var key = TestHelpers.GenerateKeyBytes();
var stringKey = TestHelpers.GenerateKeyHexString(key);
using (new TestScopedEnvironmentVariable(EnvironmentSettingNames.WebSiteAuthEncryptionKey, stringKey))
{
// Act
ObjectResult result = (ObjectResult)_hostController.GetAdminToken();
HttpStatusCode resultStatus = (HttpStatusCode)result.StatusCode;
string token = (string)result.Value;
// Assert
Assert.Equal(HttpStatusCode.OK, resultStatus);
Assert.True(SimpleWebTokenHelper.ValidateToken(token, new SystemClock()));
}
}
public async Task StandbyMode_EndToEnd_LinuxContainer()
{
byte[] bytes = TestHelpers.GenerateKeyBytes();
var encryptionKey = Convert.ToBase64String(bytes);
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.ContainerName, "TestContainer" },
{ EnvironmentSettingNames.ContainerEncryptionKey, encryptionKey },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
var httpConfig = new HttpConfiguration();
var testRootPath = Path.Combine(Path.GetTempPath(), "StandbyModeTest_Linux");
await FileUtility.DeleteDirectoryAsync(testRootPath, true);
var loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
var webHostSettings = new WebHostSettings
{
IsSelfHost = true,
LogPath = Path.Combine(testRootPath, "Logs"),
SecretsPath = Path.Combine(testRootPath, "Secrets"),
ScriptPath = Path.Combine(testRootPath, "WWWRoot")
};
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(loggerProvider);
var webHostBuilder = Program.CreateWebHostBuilder()
.ConfigureServices(c =>
{
c.AddSingleton(webHostSettings)
.AddSingleton <ILoggerProviderFactory>(loggerProviderFactory)
.AddSingleton <ILoggerFactory>(loggerFactory);
});
var httpServer = new TestServer(webHostBuilder);
var httpClient = httpServer.CreateClient();
httpClient.BaseAddress = new Uri("https://localhost/");
TestHelpers.WaitForWebHost(httpClient);
var traces = loggerProvider.GetAllLogMessages().ToArray();
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Starting Host (HostId=placeholder-host")));
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Host is in standby mode")));
// issue warmup request and verify
var request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
var response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
string responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// issue warmup request with restart and verify
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup?restart=1");
response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
responseBody = await response.Content.ReadAsStringAsync();
Assert.Equal("WarmUp complete.", responseBody);
// Now specialize the host by invoking assign
var secretManager = httpServer.Host.Services.GetService <ISecretManager>();
var masterKey = (await secretManager.GetHostSecretsAsync()).MasterKey;
string uri = "admin/instance/assign";
request = new HttpRequestMessage(HttpMethod.Post, uri);
var environment = new Dictionary <string, string>();
var assignmentContext = new HostAssignmentContext
{
SiteId = 1234,
SiteName = "TestSite",
Environment = environment
};
var encryptedAssignmentContext = EncryptedHostAssignmentContext.Create(assignmentContext, encryptionKey);
string json = JsonConvert.SerializeObject(encryptedAssignmentContext);
request.Content = new StringContent(json, Encoding.UTF8, "application/json");
request.Headers.Add(AuthenticationLevelHandler.FunctionsKeyHeaderName, masterKey);
response = await httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
// give time for the specialization to happen
string[] logLines = null;
await TestHelpers.Await(() =>
{
// wait for the trace indicating that the host has been specialized
logLines = loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
return(logLines.Contains("Generating 0 job function(s)"));
}, userMessageCallback : () => string.Join(Environment.NewLine, loggerProvider.GetAllLogMessages().Select(p => $"[{p.Timestamp.ToString("HH:mm:ss.fff")}] {p.FormattedMessage}")));
httpServer.Dispose();
httpClient.Dispose();
await Task.Delay(2000);
var hostConfig = WebHostResolver.CreateScriptHostConfiguration(webHostSettings, true);
var expectedHostId = hostConfig.HostConfig.HostId;
// verify the rest of the expected logs
string text = string.Join(Environment.NewLine, logLines);
Assert.True(logLines.Count(p => p.Contains("Stopping Host")) >= 1);
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
Assert.Equal(2, logLines.Count(p => p.Contains("Starting Host (HostId=placeholder-host")));
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Starting Host (HostId={expectedHostId}")));
Assert.Contains("Generating 0 job function(s)", logLines);
WebScriptHostManager.ResetStandbyMode();
}
}
public async Task StandbyModeE2E_LinuxContainer()
{
byte[] bytes = TestHelpers.GenerateKeyBytes();
var encryptionKey = Convert.ToBase64String(bytes);
var containerName = "testContainer";
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" },
{ EnvironmentSettingNames.ContainerName, containerName },
{ EnvironmentSettingNames.AzureWebsiteHostName, "testapp.azurewebsites.net" },
{ EnvironmentSettingNames.AzureWebsiteName, "TestApp" },
{ EnvironmentSettingNames.ContainerEncryptionKey, encryptionKey },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
{ EnvironmentSettingNames.AzureWebsiteSku, "Dynamic" },
{ EnvironmentSettingNames.AzureWebsiteZipDeployment, null },
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
};
var environment = new TestEnvironment(vars);
await InitializeTestHostAsync("Linux", environment);
// verify only the Warmup function is present
// generally when in placeholder mode, the list API won't be called
// but we're doing this for regression testing
var functions = await ListFunctions();
Assert.Equal(1, functions.Length);
Assert.Equal("WarmUp", functions[0]);
await VerifyWarmupSucceeds();
await VerifyWarmupSucceeds(restart : true);
// now specialize the site
await Assign(encryptionKey);
// immediately call a function - expect the call to block until
// the host is fully specialized
// the Unauthorized is expected since we havne't specified the key
// it's enough here to ensure we don't get a 404
var request = new HttpRequestMessage(HttpMethod.Get, $"api/httptrigger");
request.Headers.Add(ScriptConstants.AntaresColdStartHeaderName, "1");
var response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
// now that the host is initialized, send a valid key
// and expect success
var secretManager = _httpServer.Host.Services.GetService <ISecretManagerProvider>().Current;
var fd = _httpServer.Host.Services.GetService <IFunctionInvocationDispatcherFactory>();
var keys = await secretManager.GetFunctionSecretsAsync("HttpTrigger");
string key = keys.First().Value;
request = new HttpRequestMessage(HttpMethod.Get, $"api/httptrigger?code={key}");
request.Headers.Add(ScriptConstants.AntaresColdStartHeaderName, "1");
response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.False(environment.IsPlaceholderModeEnabled());
Assert.True(environment.IsContainerReady());
// verify that after specialization the correct
// app content is returned
functions = await ListFunctions();
Assert.Equal(1, functions.Length);
Assert.Equal("HttpTrigger", functions[0]);
// verify warmup function no longer there
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
string hostId = "testapp";
// verify the expected logs
var logLines = _loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
Assert.True(logLines.Count(p => p.Contains("Stopping JobHost")) >= 1);
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
Assert.Equal(1, logLines.Count(p => p.Contains("Validating host assignment context")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting Assignment")));
Assert.Equal(1, logLines.Count(p => p.Contains("Applying 3 app setting(s)")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Skipping WorkerConfig for language:python")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Skipping WorkerConfig for language:powershell")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Skipping WorkerConfig for language:java")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Extracting files to '{_expectedScriptPath}'")));
Assert.Equal(1, logLines.Count(p => p.Contains("Zip extraction complete")));
Assert.Equal(1, logLines.Count(p => p.Contains("Triggering specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
Assert.Equal(3, logLines.Count(p => p.Contains($"Starting Host (HostId={hostId}")));
Assert.Equal(3, logLines.Count(p => p.Contains($"Loading proxies metadata")));
Assert.Equal(3, logLines.Count(p => p.Contains("Initializing Azure Function proxies")));
Assert.Equal(2, logLines.Count(p => p.Contains($"1 proxies loaded")));
Assert.Equal(1, logLines.Count(p => p.Contains($"0 proxies loaded")));
Assert.Contains("Node.js HttpTrigger function invoked.", logLines);
// verify cold start log entry
var coldStartLog = _loggerProvider.GetAllLogMessages().FirstOrDefault(p => p.Category == ScriptConstants.LogCategoryHostMetrics);
JObject coldStartData = JObject.Parse(coldStartLog.FormattedMessage);
Assert.Equal("Dynamic", coldStartData["sku"]);
Assert.True((int)coldStartData["dispatchDuration"] > 0);
Assert.True((int)coldStartData["functionDuration"] > 0);
// Verify that the internal cache has reset
Assert.NotSame(GetCachedTimeZoneInfo(), _originalTimeZoneInfoCache);
}
public async Task StandbyMode_EndToEnd_LinuxContainer()
{
byte[] bytes = TestHelpers.GenerateKeyBytes();
var encryptionKey = Convert.ToBase64String(bytes);
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.ContainerName, "TestApp" },
{ EnvironmentSettingNames.AzureWebsiteName, "TestApp" },
{ EnvironmentSettingNames.ContainerEncryptionKey, encryptionKey },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
{ EnvironmentSettingNames.AzureWebsiteConfigurationReady, null },
{ EnvironmentSettingNames.AzureWebsiteSku, "Dynamic" },
{ EnvironmentSettingNames.AzureWebsiteZipDeployment, null },
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
InitializeTestHost("Linux");
await VerifyWarmupSucceeds();
await VerifyWarmupSucceeds(restart : true);
// now specialize the site
await Assign(encryptionKey);
// immediately call a function - expect the call to block until
// the host is fully specialized
// the Unauthorized is expected since we havne't specified the key
// it's enough here to ensure we don't get a 404
var request = new HttpRequestMessage(HttpMethod.Get, $"api/httptrigger");
request.Headers.Add(ScriptConstants.AntaresColdStartHeaderName, "1");
var response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
// now that the host is initialized, send a valid key
// and expect success
var secretManager = _httpServer.Host.Services.GetService <ISecretManager>();
var keys = await secretManager.GetFunctionSecretsAsync("HttpTrigger");
string key = keys.First().Value;
request = new HttpRequestMessage(HttpMethod.Get, $"api/httptrigger?code={key}");
request.Headers.Add(ScriptConstants.AntaresColdStartHeaderName, "1");
response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.False(WebScriptHostManager.InStandbyMode);
Assert.True(ScriptSettingsManager.Instance.ContainerReady);
// verify warmup function no longer there
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
_httpServer.Dispose();
_httpClient.Dispose();
// verify the expected logs
var logLines = _loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
Assert.True(logLines.Count(p => p.Contains("Stopping Host")) >= 1);
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
Assert.Equal(2, logLines.Count(p => p.Contains("Starting Host (HostId=placeholder-host")));
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
Assert.Equal(1, logLines.Count(p => p.Contains("Validating host assignment context")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting Assignment")));
Assert.Equal(1, logLines.Count(p => p.Contains("Applying 1 app setting(s)")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Extracting files to '{_webHostSettings.ScriptPath}'")));
Assert.Equal(1, logLines.Count(p => p.Contains("Zip extraction complete")));
Assert.Equal(1, logLines.Count(p => p.Contains("Triggering specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Starting Host (HostId={_expectedHostId}")));
Assert.Contains("Node.js HttpTrigger function invoked.", logLines);
// verify cold start log entry
var coldStartLog = _loggerProvider.GetAllLogMessages().FirstOrDefault(p => p.Category == ScriptConstants.LogCategoryHostMetrics);
JObject coldStartData = JObject.Parse(coldStartLog.FormattedMessage);
Assert.Equal("Dynamic", coldStartData["sku"]);
Assert.True((int)coldStartData["dispatchDuration"] > 0);
Assert.True((int)coldStartData["functionDuration"] > 0);
WebScriptHostManager.ResetStandbyMode();
}
}
public async Task StandbyMode_EndToEnd_LinuxContainer()
{
byte[] bytes = TestHelpers.GenerateKeyBytes();
var encryptionKey = Convert.ToBase64String(bytes);
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" },
{ EnvironmentSettingNames.ContainerName, "TestApp" },
{ EnvironmentSettingNames.AzureWebsiteName, "TestApp" },
{ EnvironmentSettingNames.ContainerEncryptionKey, encryptionKey },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
{ EnvironmentSettingNames.AzureWebsiteSku, "Dynamic" },
{ EnvironmentSettingNames.AzureWebsiteZipDeployment, null },
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
};
var environment = new TestEnvironment(vars);
await InitializeTestHostAsync("Linux", environment);
await VerifyWarmupSucceeds();
await VerifyWarmupSucceeds(restart : true);
// now specialize the site
await Assign(encryptionKey);
// immediately call a function - expect the call to block until
// the host is fully specialized
// the Unauthorized is expected since we havne't specified the key
// it's enough here to ensure we don't get a 404
var request = new HttpRequestMessage(HttpMethod.Get, $"api/httptrigger");
request.Headers.Add(ScriptConstants.AntaresColdStartHeaderName, "1");
var response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
// now that the host is initialized, send a valid key
// and expect success
var secretManager = _httpServer.Host.Services.GetService <ISecretManagerProvider>().Current;
var keys = await secretManager.GetFunctionSecretsAsync("HttpTrigger");
string key = keys.First().Value;
request = new HttpRequestMessage(HttpMethod.Get, $"api/httptrigger?code={key}");
request.Headers.Add(ScriptConstants.AntaresColdStartHeaderName, "1");
response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.False(environment.IsPlaceholderModeEnabled());
Assert.True(environment.IsContainerReady());
// verify warmup function no longer there
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
response = await _httpClient.SendAsync(request);
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
_httpServer.Dispose();
_httpClient.Dispose();
// make sure there are no errors
var logs = _loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null);
var error = logs.Where(p => p.Level == Microsoft.Extensions.Logging.LogLevel.Error).FirstOrDefault();
Assert.True(error == null, $"Unexpected error: {error?.FormattedMessage} - {error?.Exception.ToFormattedString()}");
string sanitizedMachineName = Environment.MachineName
.Where(char.IsLetterOrDigit)
.Aggregate(new StringBuilder(), (b, c) => b.Append(c))
.ToString().ToLowerInvariant();
// verify the expected logs
var logLines = _loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
Assert.True(logLines.Count(p => p.Contains("Stopping JobHost")) >= 1);
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
Assert.Equal(1, logLines.Count(p => p.Contains("Validating host assignment context")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting Assignment")));
Assert.Equal(1, logLines.Count(p => p.Contains("Applying 1 app setting(s)")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Extracting files to '{_expectedScriptPath}'")));
Assert.Equal(1, logLines.Count(p => p.Contains("Zip extraction complete")));
Assert.Equal(1, logLines.Count(p => p.Contains("Triggering specialization")));
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
Assert.Equal(3, logLines.Count(p => p.Contains($"Starting Host (HostId={sanitizedMachineName}")));
Assert.Equal(1, logLines.Count(p => p.Contains($"Loading proxies metadata")));
Assert.Equal(1, logLines.Count(p => p.Contains("Initializing Azure Function proxies")));
Assert.Equal(1, logLines.Count(p => p.Contains($"0 proxies loaded")));
Assert.Contains("Node.js HttpTrigger function invoked.", logLines);
// verify cold start log entry
var coldStartLog = _loggerProvider.GetAllLogMessages().FirstOrDefault(p => p.Category == ScriptConstants.LogCategoryHostMetrics);
JObject coldStartData = JObject.Parse(coldStartLog.FormattedMessage);
Assert.Equal("Dynamic", coldStartData["sku"]);
Assert.True((int)coldStartData["dispatchDuration"] > 0);
Assert.True((int)coldStartData["functionDuration"] > 0);
// Verify that the internal cache has reset
Assert.NotSame(GetCachedTimeZoneInfo(), _originalTimeZoneInfoCache);
}