public async Task Invoke(HttpContext httpContext, WebScriptHostManager manager)
{
// in standby mode, we don't want to wait for host start
bool bypassHostCheck = WebScriptHostManager.InStandbyMode;
if (!bypassHostCheck)
{
bool hostReady = await manager.DelayUntilHostReady();
if (!hostReady)
{
httpContext.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await httpContext.Response.WriteAsync("Function host is not running.");
return;
}
}
if (StandbyManager.IsWarmUpRequest(httpContext.Request))
{
await StandbyManager.WarmUp(httpContext.Request, _scriptHostManager);
}
await _next.Invoke(httpContext);
}
public void IsWarmUpRequest_ReturnsExpectedValue()
{
var request = new HttpRequestMessage(HttpMethod.Post, "http://azure.com/api/warmup");
Assert.False(StandbyManager.IsWarmUpRequest(request));
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0" },
{ EnvironmentSettingNames.AzureWebsiteInstanceId, null }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
Assert.False(StandbyManager.IsWarmUpRequest(request));
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteInstanceId, "12345");
Assert.True(StandbyManager.IsWarmUpRequest(request));
request = new HttpRequestMessage(HttpMethod.Post, "http://azure.com/api/csharphttpwarmup");
Assert.True(StandbyManager.IsWarmUpRequest(request));
request = new HttpRequestMessage(HttpMethod.Post, "http://azure.com/api/warmup");
request.Headers.Add(ScriptConstants.AntaresLogIdHeaderName, "xyz123");
Assert.False(StandbyManager.IsWarmUpRequest(request));
request = new HttpRequestMessage(HttpMethod.Post, "http://azure.com/api/foo");
Assert.False(StandbyManager.IsWarmUpRequest(request));
}
}
public StandbyManagerE2ETestBase()
{
_testRootPath = Path.Combine(Path.GetTempPath(), "StandbyManagerTests");
CleanupTestDirectory();
StandbyManager.ResetChangeToken();
}
public async Task Specialize_ResetsConfiguration()
{
var manager = new StandbyManager(_mockHostManager.Object, _mockLanguageWorkerChannelManager.Object, _mockConfiguration.Object, _mockWebHostEnvironment.Object, _testEnvironment, _mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance);
await manager.SpecializeHostAsync();
_mockConfiguration.Verify(c => c.Reload());
}
public async Task Specialize_ResetsConfiguration()
{
var hostNameProvider = new HostNameProvider(_testEnvironment, _testLoggerFactory.CreateLogger <HostNameProvider>());
var manager = new StandbyManager(_mockHostManager.Object, _mockLanguageWorkerChannelManager.Object, _mockConfiguration.Object, _mockWebHostEnvironment.Object, _testEnvironment, _mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance, hostNameProvider, _mockApplicationLifetime.Object);
await manager.SpecializeHostAsync();
_mockConfiguration.Verify(c => c.Reload());
}
public void SpecializeHost()
{
var mockScriptWebHostEnvironment = new Mock <IScriptWebHostEnvironment>();
Mock <IConfigurationRoot> mockConfiguration = new Mock <IConfigurationRoot>();
var mockEnvironment = new Mock <IEnvironment>();
Mock <ILanguageWorkerChannelManager> mockLanguageWorkerChannelManager = new Mock <ILanguageWorkerChannelManager>();
ILogger <StandbyManager> testLogger = new Logger <StandbyManager>(_loggerFactory);
var manager = new StandbyManager(_hostService, mockLanguageWorkerChannelManager.Object, mockConfiguration.Object, mockScriptWebHostEnvironment.Object, mockEnvironment.Object, _monitor, testLogger);
manager.SpecializeHostAsync().Wait();
}
public void IsWarmUpRequest_ReturnsExpectedValue()
{
var request = HttpTestHelpers.CreateHttpRequest("POST", "http://azure.com/api/warmup");
Assert.False(StandbyManager.IsWarmUpRequest(request));
var vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0" },
{ EnvironmentSettingNames.AzureWebsiteInstanceId, null }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
// in this test we're forcing a transition from non-placeholder mode to placeholder mode
// which can't happen in the wild, so we force a reset here
WebScriptHostManager.ResetStandbyMode();
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
Assert.False(StandbyManager.IsWarmUpRequest(request));
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteInstanceId, "12345");
Assert.True(StandbyManager.IsWarmUpRequest(request));
request = HttpTestHelpers.CreateHttpRequest("POST", "http://azure.com/api/csharphttpwarmup");
Assert.True(StandbyManager.IsWarmUpRequest(request));
request = HttpTestHelpers.CreateHttpRequest("POST", "http://azure.com/api/warmup");
request.Headers.Add(ScriptConstants.AntaresLogIdHeaderName, "xyz123");
Assert.False(StandbyManager.IsWarmUpRequest(request));
request = HttpTestHelpers.CreateHttpRequest("POST", "http://azure.com/api/foo");
Assert.False(StandbyManager.IsWarmUpRequest(request));
}
vars = new Dictionary <string, string>
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0" },
{ EnvironmentSettingNames.AzureWebsiteInstanceId, null }
};
using (var env = new TestScopedEnvironmentVariable(vars))
{
WebScriptHostManager.ResetStandbyMode();
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
Assert.False(StandbyManager.IsWarmUpRequest(request));
request = HttpTestHelpers.CreateHttpRequest("POST", "http://azure.com/api/warmup");
_settingsManager.SetSetting(EnvironmentSettingNames.ContainerName, "TestContainer");
Assert.True(_settingsManager.IsLinuxContainerEnvironment);
Assert.True(StandbyManager.IsWarmUpRequest(request));
}
}
public async Task Specialize_ResetsConfiguration()
{
TestMetricsLogger metricsLogger = new TestMetricsLogger();
var hostNameProvider = new HostNameProvider(_testEnvironment);
var manager = new StandbyManager(_mockHostManager.Object, _mockLanguageWorkerChannelManager.Object, _mockConfiguration.Object, _mockWebHostEnvironment.Object, _testEnvironment, _mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance, hostNameProvider, _mockApplicationLifetime.Object, metricsLogger);
await manager.SpecializeHostAsync();
// Ensure metrics are generated
Assert.True(AreExpectedMetricsGenerated(metricsLogger));
_mockConfiguration.Verify(c => c.Reload());
}
public async Task Specialize_StandbyManagerInitialize_EmitsExpectedMetric()
{
TestMetricsLogger metricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName, "placeholder.azurewebsites.net");
var hostNameProvider = new HostNameProvider(_testEnvironment);
var manager = new StandbyManager(_mockHostManager.Object, _mockLanguageWorkerChannelManager.Object, _mockConfiguration.Object, _mockWebHostEnvironment.Object, _testEnvironment, _mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance, hostNameProvider, _mockApplicationLifetime.Object, metricsLogger);
await manager.InitializeAsync().ContinueWith(t => { }); // Ignore errors.
// Ensure metric is generated
Assert.True(metricsLogger.EventsBegan.Contains(MetricEventNames.SpecializationStandbyManagerInitialize) && metricsLogger.EventsEnded.Contains(MetricEventNames.SpecializationStandbyManagerInitialize));
}
public async Task Specialize_ReloadsEnvironmentVariables()
{
_testEnvironment.SetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName, LanguageWorkerConstants.JavaLanguageWorkerName);
_mockLanguageWorkerChannelManager.Setup(m => m.SpecializeAsync()).Returns(async() =>
{
_testEnvironment.SetEnvironmentVariable(_testSettingName, _testSettingValue);
await Task.Yield();
});
_testEnvironment.SetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName, LanguageWorkerConstants.JavaLanguageWorkerName);
var manager = new StandbyManager(_mockHostManager.Object, _mockLanguageWorkerChannelManager.Object, _mockConfiguration.Object, _mockWebHostEnvironment.Object, _testEnvironment, _mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance);
await manager.SpecializeHostAsync();
Assert.Equal(_testSettingValue, _testEnvironment.GetEnvironmentVariable(_testSettingName));
}
public async Task Invoke(HttpContext httpContext)
{
if (StandbyManager.IsWarmUpRequest(httpContext.Request))
{
if (!httpContext.Items.TryGetValue(ScriptConstants.AzureFunctionsHostManagerKey, out object scriptHostManager))
{
throw new InvalidOperationException($"Warmup request received, but no instance of {nameof(WebScriptHostManager)} in HTTP context");
}
await StandbyManager.WarmUp(httpContext.Request, (WebScriptHostManager)scriptHostManager);
}
await _next.Invoke(httpContext);
}
public void SpecializeHost()
{
var mockScriptWebHostEnvironment = new Mock <IScriptWebHostEnvironment>();
Mock <IConfigurationRoot> mockConfiguration = new Mock <IConfigurationRoot>();
var mockEnvironment = new Mock <IEnvironment>();
Mock <IWebHostLanguageWorkerChannelManager> mockLanguageWorkerChannelManager = new Mock <IWebHostLanguageWorkerChannelManager>();
ILogger <StandbyManager> testLogger = new Logger <StandbyManager>(_loggerFactory);
var loggerProvider = new TestLoggerProvider();
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(loggerProvider);
var hostNameProvider = new HostNameProvider(mockEnvironment.Object, loggerFactory.CreateLogger <HostNameProvider>());
var manager = new StandbyManager(_hostService, mockLanguageWorkerChannelManager.Object, mockConfiguration.Object, mockScriptWebHostEnvironment.Object, mockEnvironment.Object, _monitor, testLogger, hostNameProvider);
manager.SpecializeHostAsync().Wait();
}
public async Task Specialize_ResetsHostNameProvider()
{
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName, "placeholder.azurewebsites.net");
var hostNameProvider = new HostNameProvider(_testEnvironment, _testLoggerFactory.CreateLogger <HostNameProvider>());
var manager = new StandbyManager(_mockHostManager.Object, _mockLanguageWorkerChannelManager.Object, _mockConfiguration.Object, _mockWebHostEnvironment.Object, _testEnvironment, _mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance, hostNameProvider, _mockApplicationLifetime.Object);
Assert.Equal("placeholder.azurewebsites.net", hostNameProvider.Value);
await manager.SpecializeHostAsync();
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName, "testapp.azurewebsites.net");
Assert.Equal("testapp.azurewebsites.net", hostNameProvider.Value);
_mockConfiguration.Verify(c => c.Reload());
}
public void SpecializeHost()
{
var mockScriptWebHostEnvironment = new Mock <IScriptWebHostEnvironment>();
Mock <IConfigurationRoot> mockConfiguration = new Mock <IConfigurationRoot>();
var mockEnvironment = new Mock <IEnvironment>();
Mock <IWebHostRpcWorkerChannelManager> mockLanguageWorkerChannelManager = new Mock <IWebHostRpcWorkerChannelManager>();
ILogger <StandbyManager> testLogger = new Logger <StandbyManager>(_loggerFactory);
var loggerProvider = new TestLoggerProvider();
var loggerFactory = new LoggerFactory();
loggerFactory.AddProvider(loggerProvider);
var hostNameProvider = new HostNameProvider(mockEnvironment.Object);
var mockApplicationLifetime = new Mock <Microsoft.AspNetCore.Hosting.IApplicationLifetime>(MockBehavior.Strict);
var manager = new StandbyManager(_hostService, mockLanguageWorkerChannelManager.Object, mockConfiguration.Object, mockScriptWebHostEnvironment.Object, mockEnvironment.Object, _monitor, testLogger, hostNameProvider, mockApplicationLifetime.Object, new TestMetricsLogger());
manager.SpecializeHostAsync().Wait();
}
public SpecializationE2ETests()
{
StandbyManager.ResetChangeToken();
var settings = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
};
_environment = new TestEnvironment(settings);
_loggerProvider = new TestLoggerProvider();
_pauseBeforeHostBuild = new SemaphoreSlim(1, 1);
_pauseAfterStandbyHostBuild = new SemaphoreSlim(1, 1);
_buildCount = new SemaphoreSlim(2, 2);
}
public async Task Specialize_ResetsConfiguration()
{
var mockHostManager = new Mock <IScriptHostManager>();
mockHostManager.Setup(m => m.State)
.Returns(ScriptHostState.Running);
var mockConfiguration = new Mock <IConfigurationRoot>();
var mockOptionsMonitor = new Mock <IOptionsMonitor <ScriptApplicationHostOptions> >();
var mockWebHostEnvironment = new Mock <IScriptWebHostEnvironment>();
var mockEnvironment = new TestEnvironment();
var manager = new StandbyManager(mockHostManager.Object, mockConfiguration.Object, mockWebHostEnvironment.Object, mockEnvironment, mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance);
await manager.SpecializeHostAsync();
mockConfiguration.Verify(c => c.Reload());
}
public async Task Specialize_ReloadsEnvironmentVariables()
{
TestMetricsLogger metricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.JavaLanguageWorkerName);
_mockLanguageWorkerChannelManager.Setup(m => m.SpecializeAsync()).Returns(async() =>
{
_testEnvironment.SetEnvironmentVariable(_testSettingName, _testSettingValue);
await Task.Yield();
});
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.JavaLanguageWorkerName);
var hostNameProvider = new HostNameProvider(_testEnvironment);
var manager = new StandbyManager(_mockHostManager.Object, _mockLanguageWorkerChannelManager.Object, _mockConfiguration.Object, _mockWebHostEnvironment.Object, _testEnvironment, _mockOptionsMonitor.Object, NullLogger <StandbyManager> .Instance, hostNameProvider, _mockApplicationLifetime.Object, metricsLogger);
await manager.SpecializeHostAsync();
// Ensure metrics are generated
Assert.True(AreExpectedMetricsGenerated(metricsLogger));
Assert.Equal(_testSettingValue, _testEnvironment.GetEnvironmentVariable(_testSettingName));
}
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
SetRequestId(request);
var resolver = _config.DependencyResolver;
var scriptHostManager = resolver.GetService <WebScriptHostManager>();
if (!scriptHostManager.Initialized)
{
scriptHostManager.Initialize();
}
var webHostSettings = resolver.GetService <WebHostSettings>();
if (webHostSettings.IsAuthDisabled)
{
request.SetProperty(ScriptConstants.AzureFunctionsHttpRequestAuthorizationDisabledKey, true);
}
// some routes do not require the host to be running (most do)
bool bypassHostCheck = request.MatchRoute("admin/host/status");
if (!bypassHostCheck)
{
// If the host is not running, we'll wait a bit for it to fully
// initialize. This might happen if http requests come in while the
// host is starting up for the first time, or if it is restarting.
await scriptHostManager.DelayUntilHostReady(_hostTimeoutSeconds, _hostRunningPollIntervalMilliseconds);
}
if (StandbyManager.IsWarmUpRequest(request))
{
await StandbyManager.WarmUp(request, scriptHostManager);
}
return(await base.SendAsync(request, cancellationToken));
}
public async Task InvocationsContainDifferentOperationIds()
{
// Verify that when a request specializes the host we don't capture the context
// of that request. Application Insights uses this context to correlate telemetry
// so it had a confusing effect. Previously all TimerTrigger traces would have the
// operation id of this request and all host logs would as well.
// Start a host in standby mode.
StandbyManager.ResetChangeToken();
string standbyPath = Path.Combine(Path.GetTempPath(), "functions", "standby", "wwwroot");
string specializedScriptRoot = @"TestScripts\CSharp";
string scriptRootConfigPath = ConfigurationPath.Combine(ConfigurationSectionNames.WebHost, nameof(ScriptApplicationHostOptions.ScriptPath));
var settings = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
};
var environment = new TestEnvironment(settings);
var loggerProvider = new TestLoggerProvider();
var channel = new TestTelemetryChannel();
var builder = Program.CreateWebHostBuilder()
.ConfigureLogging(b =>
{
b.AddProvider(loggerProvider);
})
.ConfigureAppConfiguration(c =>
{
c.AddInMemoryCollection(new Dictionary <string, string>
{
{ scriptRootConfigPath, specializedScriptRoot }
});
})
.ConfigureServices((bc, s) =>
{
s.AddSingleton <IEnvironment>(environment);
// Ensure that we don't have a race between the timer and the
// request for triggering specialization.
s.AddSingleton <IStandbyManager, InfiniteTimerStandbyManager>();
})
.AddScriptHostBuilder(webJobsBuilder =>
{
webJobsBuilder.Services.AddSingleton <ITelemetryChannel>(_ => channel);
webJobsBuilder.Services.Configure <FunctionResultAggregatorOptions>(o =>
{
o.IsEnabled = false;
});
webJobsBuilder.Services.PostConfigure <ApplicationInsightsLoggerOptions>(o =>
{
o.SamplingSettings = null;
});
webJobsBuilder.Services.PostConfigure <ScriptJobHostOptions>(o =>
{
// Only load the function we care about, but not during standby
if (o.RootScriptPath != standbyPath)
{
o.Functions = new[]
{
"OneSecondTimer",
"FunctionExecutionContext"
};
}
});
})
.ConfigureScriptHostAppConfiguration(c =>
{
c.AddInMemoryCollection(new Dictionary <string, string>
{
[EnvironmentSettingNames.AppInsightsInstrumentationKey] = "some_key"
});
});
using (var testServer = new TestServer(builder))
{
var client = testServer.CreateClient();
HttpResponseMessage response = await client.GetAsync("api/warmup");
Assert.True(response.IsSuccessStatusCode, loggerProvider.GetLog());
// Now that standby mode is warmed up, set the specialization properties...
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteContainerReady, "1");
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
// ...and issue a request which will force specialization.
response = await client.GetAsync("api/functionexecutioncontext");
Assert.True(response.IsSuccessStatusCode, loggerProvider.GetLog());
// Wait until we have a few logs from the timer trigger.
IEnumerable <TraceTelemetry> timerLogs = null;
await TestHelpers.Await(() =>
{
timerLogs = channel.Telemetries
.OfType <TraceTelemetry>()
.Where(p => p.Message == "OneSecondTimer fired!");
return(timerLogs.Count() >= 3);
});
var startupRequest = channel.Telemetries
.OfType <RequestTelemetry>()
.Where(p => p.Name == "FunctionExecutionContext")
.Single();
// Make sure that auto-Http tracking worked with this request.
Assert.Equal("200", startupRequest.ResponseCode);
// The host logs should not be associated with this request.
var logsWithRequestId = channel.Telemetries
.OfType <TraceTelemetry>()
.Select(p => p.Context.Operation.Id)
.Where(p => p == startupRequest.Context.Operation.Id);
// Just expect the "Executing" and "Executed" logs from the actual request.
Assert.Equal(2, logsWithRequestId.Count());
// And each of the timer invocations should have a different operation id, and none
// should match the request id.
var distinctOpIds = timerLogs.Select(p => p.Context.Operation.Id).Distinct();
Assert.Equal(timerLogs.Count(), distinctOpIds.Count());
Assert.Empty(timerLogs.Where(p => p.Context.Operation.Id == startupRequest.Context.Operation.Id));
}
}
public async Task IsPlaceholderMode_ThroughoutInitialization_EvaluatesCorrectly()
{
StandbyManager.ResetChangeToken();
string standbyPath = Path.Combine(Path.GetTempPath(), "functions", "standby", "wwwroot");
string specializedScriptRoot = @"TestScripts\CSharp";
string scriptRootConfigPath = ConfigurationPath.Combine(ConfigurationSectionNames.WebHost, nameof(ScriptApplicationHostOptions.ScriptPath));
var settings = new Dictionary <string, string>()
{
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1" },
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
};
var environment = new TestEnvironment(settings);
var loggerProvider = new TestLoggerProvider();
var builder = Program.CreateWebHostBuilder()
.ConfigureLogging(b =>
{
b.AddProvider(loggerProvider);
})
.ConfigureAppConfiguration(c =>
{
c.AddInMemoryCollection(new Dictionary <string, string>
{
{ scriptRootConfigPath, specializedScriptRoot }
});
})
.ConfigureServices((bc, s) =>
{
s.AddSingleton <IEnvironment>(environment);
// Simulate the environment becoming specialized after these options have been
// initialized with standby paths.
s.AddOptions <ScriptApplicationHostOptions>()
.PostConfigure <IEnvironment>((o, e) =>
{
Specialize(e);
});
})
.AddScriptHostBuilder(webJobsBuilder =>
{
webJobsBuilder.Services.PostConfigure <ScriptJobHostOptions>(o =>
{
// Only load the function we care about, but not during standby
if (o.RootScriptPath != standbyPath)
{
o.Functions = new[]
{
"HttpTrigger-Dynamic"
};
}
});
});
var server = new TestServer(builder);
var client = server.CreateClient();
// Force the specialization middleware to run
HttpResponseMessage response = await InvokeFunction(client);
response.EnsureSuccessStatusCode();
string log = loggerProvider.GetLog();
Assert.Contains("Creating StandbyMode placeholder function directory", log);
Assert.Contains("Starting host specialization", log);
// Make sure this was registered.
var hostedServices = server.Host.Services.GetServices <IHostedService>();
Assert.Contains(hostedServices, p => p is StandbyInitializationService);
}
