public async Task OnTimeoutExceptionAsync(ExceptionDispatchInfo exceptionInfo, TimeSpan timeoutGracePeriod) { FunctionTimeoutException timeoutException = exceptionInfo.SourceException as FunctionTimeoutException; if (timeoutException?.Task != null) { // We may double the timeoutGracePeriod here by first waiting to see if the iniital // function task that started the exception has completed. Task completedTask = await Task.WhenAny(timeoutException.Task, Task.Delay(timeoutGracePeriod)); // If the function task has completed, simply return. The host has already logged the timeout. if (completedTask == timeoutException.Task) { return; } } // We can't wait on this as it may cause a deadlock if the timeout was fired // by a Listener that cannot stop until it has completed. // TODO: DI (FACAVAL) The shutdown call will invoke the host stop... but we may need to do this // explicitly in order to pass the timeout. // Task ignoreTask = _hostManager.StopAsync(); // Give the manager and all running tasks some time to shut down gracefully. //await Task.Delay(timeoutGracePeriod); IFunctionInvocationDispatcher functionInvocationDispatcher = _functionInvocationDispatcherFactory.GetFunctionDispatcher(); if (!functionInvocationDispatcher.State.Equals(FunctionInvocationDispatcherState.Default)) { _logger.LogWarning($"A function timeout has occurred. Restarting worker process executing invocationId '{timeoutException.InstanceId}'.", exceptionInfo.SourceException); // If invocation id is not found in any of the workers => worker is already disposed. No action needed. await functionInvocationDispatcher.RestartWorkerWithInvocationIdAsync(timeoutException.InstanceId.ToString()); _logger.LogWarning("Restart of language worker process(es) completed.", exceptionInfo.SourceException); } else { LogErrorAndFlush("A function timeout has occurred. Host is shutting down.", exceptionInfo.SourceException); _applicationLifetime.StopApplication(); } }
// Specify the "builtin binding types". These are types that are directly accesible without needing an explicit load gesture. // This is the set of bindings we shipped prior to binding extensibility. // Map from BindingType to the Assembly Qualified Type name for its IExtensionConfigProvider object. public ScriptHost(IOptions <JobHostOptions> options, IOptions <HttpWorkerOptions> httpWorkerOptions, IEnvironment environment, IJobHostContextFactory jobHostContextFactory, IConfiguration configuration, IDistributedLockManager distributedLockManager, IScriptEventManager eventManager, ILoggerFactory loggerFactory, IFunctionInvocationDispatcherFactory functionDispatcherFactory, IFunctionMetadataManager functionMetadataManager, IFileLoggingStatusManager fileLoggingStatusManager, IMetricsLogger metricsLogger, IOptions <ScriptJobHostOptions> scriptHostOptions, ITypeLocator typeLocator, IScriptHostManager scriptHostManager, IDebugStateProvider debugManager, IEnumerable <IScriptBindingProvider> bindingProviders, IPrimaryHostStateProvider primaryHostStateProvider, IJobHostMetadataProvider metadataProvider, IHostIdProvider hostIdProvider, IHttpRoutesManager httpRoutesManager, IApplicationLifetime applicationLifetime, IExtensionBundleManager extensionBundleManager, ScriptSettingsManager settingsManager = null) : base(options, jobHostContextFactory) { _environment = environment; _typeLocator = typeLocator as ScriptTypeLocator ?? throw new ArgumentException(nameof(typeLocator), $"A {nameof(ScriptTypeLocator)} instance is required."); _instanceId = Guid.NewGuid().ToString(); _hostOptions = options; _configuration = configuration; _storageConnectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage); _distributedLockManager = distributedLockManager; _functionMetadataManager = functionMetadataManager; _fileLoggingStatusManager = fileLoggingStatusManager; _applicationLifetime = applicationLifetime; _hostIdProvider = hostIdProvider; _httpRoutesManager = httpRoutesManager; _isHttpWorker = httpWorkerOptions.Value.Description != null; ScriptOptions = scriptHostOptions.Value; _scriptHostManager = scriptHostManager; FunctionErrors = new Dictionary <string, ICollection <string> >(StringComparer.OrdinalIgnoreCase); EventManager = eventManager; _functionDispatcher = functionDispatcherFactory.GetFunctionDispatcher(); _settingsManager = settingsManager ?? ScriptSettingsManager.Instance; ExtensionBundleManager = extensionBundleManager; _metricsLogger = metricsLogger; _hostLogPath = Path.Combine(ScriptOptions.RootLogPath, "Host"); _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); _loggerFactory = loggerFactory; _logger = loggerFactory.CreateLogger(LogCategories.Startup); Logger = _logger; _debugManager = debugManager; _primaryHostStateProvider = primaryHostStateProvider; _bindingProviders = new List <IScriptBindingProvider>(bindingProviders); _metadataProvider = metadataProvider; _eventSubscriptions.Add(EventManager.OfType <FunctionIndexingEvent>() .Subscribe(evt => { HandleHostError(evt.Exception); })); }
public FunctionInvocationDispatcherShutdownManager(IFunctionInvocationDispatcherFactory functionDispatcherFactory) { _functionDispatcher = functionDispatcherFactory.GetFunctionDispatcher(); }