public IActionResult DrainStatus([FromServices] IScriptHostManager scriptHostManager)
{
if (Utility.TryGetHostService(scriptHostManager, out IFunctionActivityStatusProvider functionActivityStatusProvider) &&
Utility.TryGetHostService(scriptHostManager, out IDrainModeManager drainModeManager))
{
var functionActivityStatus = functionActivityStatusProvider.GetStatus();
DrainModeState state = DrainModeState.Disabled;
if (drainModeManager.IsDrainModeEnabled)
{
state = (functionActivityStatus.OutstandingInvocations == 0 && functionActivityStatus.OutstandingRetries == 0)
? DrainModeState.Completed : DrainModeState.InProgress;
}
DrainModeStatus status = new DrainModeStatus()
{
State = state
};
string message = $"Drain Status: {JsonConvert.SerializeObject(state, Formatting.Indented)}, Activity Status: {JsonConvert.SerializeObject(functionActivityStatus, Formatting.Indented)}";
_logger.LogDebug(message);
return(Ok(status));
}
else
{
return(StatusCode(StatusCodes.Status503ServiceUnavailable));
}
}
public async Task <IActionResult> Drain([FromServices] IScriptHostManager scriptHostManager)
{
_logger.LogDebug("Received request to drain the host");
if (!Utility.TryGetHostService(scriptHostManager, out IDrainModeManager drainModeManager))
{
return(StatusCode(StatusCodes.Status503ServiceUnavailable));
}
await _drainSemaphore.WaitAsync();
// Stop call to some listeners gets stuck, not waiting for the stop call to complete
_ = drainModeManager.EnableDrainModeAsync(CancellationToken.None)
.ContinueWith(
antecedent =>
{
if (antecedent.Status == TaskStatus.Faulted)
{
_logger.LogError(antecedent.Exception, "Something went wrong invoking drain mode");
}
_drainSemaphore.Release();
});
return(Accepted());
}
public async Task <IActionResult> Resume([FromServices] IScriptHostManager scriptHostManager)
{
try
{
await _resumeSemaphore.WaitAsync();
ScriptHostState currentState = scriptHostManager.State;
_logger.LogDebug($"Received request to resume a draining host - host status: {currentState.ToString()}");
if (currentState != ScriptHostState.Running ||
!Utility.TryGetHostService(scriptHostManager, out IDrainModeManager drainModeManager))
{
_logger.LogDebug("The host is not in a state where we can resume.");
return(StatusCode(StatusCodes.Status409Conflict));
}
_logger.LogDebug($"Drain mode enabled: {drainModeManager.IsDrainModeEnabled}");
if (drainModeManager.IsDrainModeEnabled)
{
_logger.LogDebug("Starting a new host");
await scriptHostManager.RestartHostAsync();
}
var status = new ResumeStatus {
State = scriptHostManager.State
};
return(Ok(status));
}
finally
{
_resumeSemaphore.Release();
}
}
public async Task Invoke(HttpContext httpContext, IScriptHostManager scriptHostManager)
{
if (scriptHostManager.State != ScriptHostState.Offline)
{
using (Logger.VerifyingHostAvailabilityScope(_logger, httpContext.TraceIdentifier))
{
Logger.InitiatingHostAvailabilityCheck(_logger);
bool hostReady = await scriptHostManager.DelayUntilHostReady();
if (!hostReady)
{
Logger.HostUnavailableAfterCheck(_logger);
httpContext.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await httpContext.Response.WriteAsync("Function host is not running.");
return;
}
Logger.HostAvailabilityCheckSucceeded(_logger);
await _next.Invoke(httpContext);
}
}
else
{
// host is offline so return the app_offline.htm file content
var offlineFilePath = Path.Combine(_applicationHostOptions.CurrentValue.ScriptPath, ScriptConstants.AppOfflineFileName);
httpContext.Response.ContentType = "text/html";
httpContext.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await httpContext.Response.SendFileAsync(offlineFilePath);
}
}
public FileMonitoringService(IOptions <ScriptJobHostOptions> scriptOptions, ILoggerFactory loggerFactory, IScriptEventManager eventManager, IApplicationLifetime applicationLifetime, IScriptHostManager scriptHostManager, IEnvironment environment)
{
_scriptOptions = scriptOptions.Value;
_eventManager = eventManager;
_applicationLifetime = applicationLifetime;
_scriptHostManager = scriptHostManager;
_hostLogPath = Path.Combine(_scriptOptions.RootLogPath, "Host");
_logger = loggerFactory.CreateLogger(LogCategories.Startup);
_environment = environment;
// Use this for newer logs as we can't change existing categories of log messages
_typedLogger = loggerFactory.CreateLogger <FileMonitoringService>();
// If a file change should result in a restart, we debounce the event to
// ensure that only a single restart is triggered within a specific time window.
// This allows us to deal with a large set of file change events that might
// result from a bulk copy/unzip operation. In such cases, we only want to
// restart after ALL the operations are complete and there is a quiet period.
_restart = RestartAsync;
_restart = _restart.Debounce(500);
_shutdown = Shutdown;
_shutdown = _shutdown.Debounce(milliseconds: 500);
_rootDirectorySnapshot = GetDirectorySnapshot();
}
public StandbyManager(IScriptHostManager scriptHostManager, IOptionsMonitor <ScriptApplicationHostOptions> options, ILoggerFactory loggerFactory)
{
_scriptHostManager = scriptHostManager ?? throw new ArgumentNullException(nameof(scriptHostManager));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = loggerFactory.CreateLogger(LogCategories.Startup);
_specializationTask = new Lazy <Task>(SpecializeHostCoreAsync, LazyThreadSafetyMode.ExecutionAndPublication);
}
public async Task <IActionResult> Warmup([FromServices] IScriptHostManager scriptHostManager)
{
// Endpoint only for Windows Elastic Premium or Linux App Service plans
if (!(_environment.IsLinuxAppService() || _environment.IsWindowsElasticPremium()))
{
return(BadRequest("This API is not available for the current hosting plan"));
}
if (Interlocked.CompareExchange(ref _warmupExecuted, 1, 0) != 0)
{
return(Ok());
}
if (scriptHostManager is IServiceProvider serviceProvider)
{
IScriptJobHost jobHost = serviceProvider.GetService <IScriptJobHost>();
if (jobHost == null)
{
_logger.LogError($"No active host available.");
return(StatusCode(503));
}
await jobHost.TryInvokeWarmupAsync();
return(Ok());
}
return(BadRequest("This API is not supported by the current hosting environment."));
}
public HostWarmupMiddleware(RequestDelegate next, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, IScriptHostManager hostManager)
{
_next = next;
_webHostEnvironment = webHostEnvironment;
_environment = environment;
_hostManager = hostManager;
}
public async Task Invoke(HttpContext httpContext, IScriptHostManager scriptHostManager)
{
if (scriptHostManager.State != ScriptHostState.Offline)
{
using (Logger.VerifyingHostAvailabilityScope(_logger, httpContext.TraceIdentifier))
{
Logger.InitiatingHostAvailabilityCheck(_logger);
bool hostReady = await scriptHostManager.DelayUntilHostReady();
if (!hostReady)
{
Logger.HostUnavailableAfterCheck(_logger);
httpContext.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
await httpContext.Response.WriteAsync("Function host is not running.");
return;
}
Logger.HostAvailabilityCheckSucceeded(_logger);
await _next.Invoke(httpContext);
}
}
else
{
await httpContext.SetOfflineResponseAsync(_applicationHostOptions.CurrentValue.ScriptPath);
}
}
public InfiniteTimerStandbyManager(IScriptHostManager scriptHostManager, IWebHostRpcWorkerChannelManager rpcWorkerChannelManager,
IConfiguration configuration, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment,
IOptionsMonitor <ScriptApplicationHostOptions> options, ILogger <StandbyManager> logger, HostNameProvider hostNameProvider, IApplicationLifetime applicationLifetime)
: base(scriptHostManager, rpcWorkerChannelManager, configuration, webHostEnvironment, environment, options,
logger, hostNameProvider, applicationLifetime, TimeSpan.FromMilliseconds(-1), new TestMetricsLogger())
{
}
public HostAvailabilityCheckMiddleware(RequestDelegate next, ILoggerFactory loggerFactory, IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, IScriptHostManager scriptHostManager)
{
_next = next;
_logger = loggerFactory.CreateLogger <HostAvailabilityCheckMiddleware>();
_applicationHostOptions = applicationHostOptions;
_scriptHostManager = scriptHostManager;
}
public HostAzureBlobStorageProvider(IScriptHostManager scriptHostManager, IConfiguration configuration, IOptionsMonitor <JobHostInternalStorageOptions> options, ILogger <HostAzureBlobStorageProvider> logger, AzureComponentFactory componentFactory, AzureEventSourceLogForwarder logForwarder)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
_storageOptions = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_blobServiceClientProvider = new BlobServiceClientProvider(componentFactory, logForwarder);
if (FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableMergedWebHostScriptHostConfiguration))
{
Configuration = configuration;
}
else
{
if (scriptHostManager == null)
{
throw new ArgumentNullException(nameof(scriptHostManager));
}
Configuration = new ConfigurationBuilder()
.Add(new ActiveHostConfigurationSource(scriptHostManager))
.AddConfiguration(configuration)
.Build();
}
}
public async Task <IActionResult> GetHostStatus([FromServices] IScriptHostManager scriptHostManager, [FromServices] IHostIdProvider hostIdProvider)
{
var status = new HostStatus
{
State = scriptHostManager.State.ToString(),
Version = ScriptHost.Version,
VersionDetails = Utility.GetInformationalVersion(typeof(ScriptHost)),
Id = await hostIdProvider.GetHostIdAsync(CancellationToken.None),
ProcessUptime = (long)(DateTime.UtcNow - Process.GetCurrentProcess().StartTime).TotalMilliseconds
};
var lastError = scriptHostManager.LastError;
if (lastError != null)
{
status.Errors = new Collection <string>();
status.Errors.Add(Utility.FlattenException(lastError));
}
string message = $"Host Status: {JsonConvert.SerializeObject(status, Formatting.Indented)}";
_logger.LogInformation(message);
return(Ok(status));
}
public InfiniteTimerStandbyManager(IScriptHostManager scriptHostManager, IWebHostLanguageWorkerChannelManager languageWorkerChannelManager,
IConfiguration configuration, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment,
IOptionsMonitor <ScriptApplicationHostOptions> options, ILogger <StandbyManager> logger, HostNameProvider hostNameProvider)
: base(scriptHostManager, languageWorkerChannelManager, configuration, webHostEnvironment, environment, options,
logger, hostNameProvider, TimeSpan.FromMilliseconds(-1))
{
}
public HostWarmupMiddleware(RequestDelegate next, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, IScriptHostManager hostManager, ILogger <HostWarmupMiddleware> logger)
{
_next = next;
_webHostEnvironment = webHostEnvironment;
_environment = environment;
_hostManager = hostManager;
_logger = logger;
_assemblyLocalPath = Path.GetDirectoryName(new Uri(typeof(HostWarmupMiddleware).Assembly.Location).LocalPath);
}
public KeysController(IOptions <ScriptApplicationHostOptions> applicationOptions, ISecretManagerProvider secretManagerProvider, ILoggerFactory loggerFactory, IFileSystem fileSystem, IFunctionsSyncManager functionsSyncManager, IScriptHostManager hostManager)
{
_applicationOptions = applicationOptions;
_secretManagerProvider = secretManagerProvider;
_logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryKeysController);
_fileSystem = fileSystem;
_functionsSyncManager = functionsSyncManager;
_hostManager = hostManager;
}
public async Task <IActionResult> Disable([FromServices] IScriptHostManager hostManager)
{
_logger.LogDebug("Disabling container");
// Mark the container disabled. We check for this on host restart
await Utility.MarkContainerDisabled(_logger);
var tIgnore = Task.Run(() => hostManager.RestartHostAsync());
return(Ok());
}
public FunctionsSyncService(ILoggerFactory loggerFactory, IScriptHostManager scriptHostManager, IPrimaryHostStateProvider primaryHostStateProvider, IFunctionsSyncManager functionsSyncManager)
{
ArgumentNullException.ThrowIfNull(loggerFactory);
DueTime = 30 * 1000;
_scriptHostManager = scriptHostManager;
_primaryHostStateProvider = primaryHostStateProvider;
_functionsSyncManager = functionsSyncManager;
_logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
}
public IActionResult GetConfig([FromServices] IScriptHostManager scriptHostManager)
{
if (Utility.TryGetHostService(scriptHostManager, out IHostOptionsProvider provider))
{
return(Ok(provider.GetOptions().ToString()));
}
else
{
return(StatusCode(StatusCodes.Status503ServiceUnavailable));
}
}
public StandbyManager(IScriptHostManager scriptHostManager, ILanguageWorkerChannelManager languageWorkerChannelManager, IConfiguration configuration, IScriptWebHostEnvironment webHostEnvironment,
IEnvironment environment, IOptionsMonitor <ScriptApplicationHostOptions> options, ILogger <StandbyManager> logger)
{
_scriptHostManager = scriptHostManager ?? throw new ArgumentNullException(nameof(scriptHostManager));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_specializationTask = new Lazy <Task>(SpecializeHostCoreAsync, LazyThreadSafetyMode.ExecutionAndPublication);
_webHostEnvironment = webHostEnvironment ?? throw new ArgumentNullException(nameof(webHostEnvironment));
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
_configuration = configuration as IConfigurationRoot ?? throw new ArgumentNullException(nameof(configuration));
_languageWorkerChannelManager = languageWorkerChannelManager ?? throw new ArgumentNullException(nameof(languageWorkerChannelManager));
}
public ActiveHostWebJobsOptionsSetup(IScriptHostManager scriptHostManager, IConfiguration configuration)
{
if (scriptHostManager == null)
{
throw new ArgumentNullException(nameof(scriptHostManager));
}
_configuration = new ConfigurationBuilder()
.Add(new ActiveHostConfigurationSource(scriptHostManager))
.AddConfiguration(configuration)
.Build();
}
public IActionResult Ping([FromServices] IScriptHostManager scriptHostManager)
{
var pingStatus = new JObject
{
{ "hostState", scriptHostManager.State.ToString() }
};
string message = $"Ping Status: {pingStatus.ToString()}";
_logger.Log(LogLevel.Debug, new EventId(0, "PingStatus"), message);
return(Ok());
}
public HostController(IOptions <ScriptApplicationHostOptions> applicationHostOptions,
ILoggerFactory loggerFactory,
IEnvironment environment,
IScriptHostManager scriptHostManager,
IFunctionsSyncManager functionsSyncManager,
HostPerformanceManager performanceManager)
{
_applicationHostOptions = applicationHostOptions;
_logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostController);
_environment = environment;
_scriptHostManager = scriptHostManager;
_functionsSyncManager = functionsSyncManager;
_performanceManager = performanceManager;
}
public StandbyManager(IScriptHostManager scriptHostManager, ILanguageWorkerChannelManager languageWorkerChannelManager, IConfiguration configuration, IScriptWebHostEnvironment webHostEnvironment,
IEnvironment environment, IOptionsMonitor <ScriptApplicationHostOptions> options, ILogger <StandbyManager> logger, HostNameProvider hostNameProvider)
{
_scriptHostManager = scriptHostManager ?? throw new ArgumentNullException(nameof(scriptHostManager));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_specializationTask = new Lazy <Task>(SpecializeHostCoreAsync, LazyThreadSafetyMode.ExecutionAndPublication);
_webHostEnvironment = webHostEnvironment ?? throw new ArgumentNullException(nameof(webHostEnvironment));
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
_configuration = configuration as IConfigurationRoot ?? throw new ArgumentNullException(nameof(configuration));
_languageWorkerChannelManager = languageWorkerChannelManager ?? throw new ArgumentNullException(nameof(languageWorkerChannelManager));
_hostNameProvider = hostNameProvider ?? throw new ArgumentNullException(nameof(hostNameProvider));
_changeTokenCallbackSubscription = ChangeToken.RegisterChangeCallback(_ => _logger.LogDebug($"{nameof(StandbyManager)}.{nameof(ChangeToken)} callback has fired."), null);
}
internal static bool TryGetHostService <TService>(IScriptHostManager scriptHostManager, out TService service) where TService : class
{
service = null;
try
{
service = (scriptHostManager as IServiceProvider)?.GetService <TService>();
}
catch
{
// can get exceptions if the host is being disposed
}
return(service != null);
}
public async Task Invoke(HttpContext httpContext, IScriptHostManager manager, ILogger <TestMiddleware> logger)
{
if (httpContext.Request.Query.ContainsKey("reset"))
{
await manager.RestartHostAsync(CancellationToken.None);
logger.LogWarning("Environment reset");
var features = httpContext.Features;
var servicesFeature = features.Get <IServiceProvidersFeature>();
features.Set <IServiceProvidersFeature>(new RequestServicesFeature(httpContext, _scopeFactory));
}
await _next(httpContext);
}
public HostController(IOptions <ScriptApplicationHostOptions> applicationHostOptions,
IOptions <JobHostOptions> hostOptions,
ILoggerFactory loggerFactory,
IAuthorizationService authorizationService,
IWebFunctionsManager functionsManager,
IEnvironment environment,
IScriptHostManager scriptHostManager)
{
_applicationHostOptions = applicationHostOptions;
_hostOptions = hostOptions;
_logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostController);
_authorizationService = authorizationService;
_functionsManager = functionsManager;
_environment = environment;
_scriptHostManager = scriptHostManager;
}
public static async Task <bool> DelayUntilHostReady(this IScriptHostManager hostManager, int timeoutSeconds = ScriptConstants.HostTimeoutSeconds, int pollingIntervalMilliseconds = ScriptConstants.HostPollingIntervalMilliseconds)
{
if (CanInvoke(hostManager))
{
return(true);
}
else
{
await Utility.DelayAsync(timeoutSeconds, pollingIntervalMilliseconds, () =>
{
return(!CanInvoke(hostManager) && hostManager.State != ScriptHostState.Error);
});
return(CanInvoke(hostManager));
}
}
public static async Task <bool> DelayUntilHostReady(this IScriptHostManager hostManager, int timeoutSeconds = ScriptConstants.HostTimeoutSeconds, int pollingIntervalMilliseconds = ScriptConstants.HostPollingIntervalMilliseconds)
{
bool CanInvoke()
{
return(hostManager.State == ScriptHostState.Running || hostManager.State == ScriptHostState.Initialized);
}
await Utility.DelayAsync(timeoutSeconds, pollingIntervalMilliseconds, () =>
{
return(!CanInvoke() &&
hostManager.State != ScriptHostState.Error);
});
bool hostReady = CanInvoke();
return(hostReady);
}
public async Task <IActionResult> GetHostStatus([FromServices] IScriptHostManager scriptHostManager, [FromServices] IHostIdProvider hostIdProvider, [FromServices] IServiceProvider serviceProvider = null)
{
// When making changes to HostStatus, ensure that you update the HostStatus class in AAPT-Antares-Websites repo as well.
var status = new HostStatus
{
State = scriptHostManager.State.ToString(),
Version = ScriptHost.Version,
VersionDetails = Utility.GetInformationalVersion(typeof(ScriptHost)),
PlatformVersion = _environment.GetAntaresVersion(),
InstanceId = _environment.GetInstanceId(),
ComputerName = _environment.GetAntaresComputerName(),
Id = await hostIdProvider.GetHostIdAsync(CancellationToken.None),
ProcessUptime = (long)(DateTime.UtcNow - Process.GetCurrentProcess().StartTime).TotalMilliseconds,
FunctionAppContentEditingState = Utility.GetFunctionAppContentEditingState(_environment, _applicationHostOptions)
};
var bundleManager = serviceProvider.GetService <IExtensionBundleManager>();
if (bundleManager != null)
{
var bundleInfo = await bundleManager.GetExtensionBundleDetails();
if (bundleInfo != null)
{
status.ExtensionBundle = new Models.ExtensionBundle()
{
Id = bundleInfo.Id,
Version = bundleInfo.Version
};
}
}
var lastError = scriptHostManager.LastError;
if (lastError != null)
{
status.Errors = new Collection <string>();
status.Errors.Add(Utility.FlattenException(lastError));
}
string message = $"Host Status: {JsonConvert.SerializeObject(status, Formatting.Indented)}";
_logger.LogInformation(message);
return(Ok(status));
}
