public WorkerFunctionDescriptorProvider(ScriptHost host, ScriptJobHostOptions config, ICollection <IScriptBindingProvider> bindingProviders,
IFunctionInvocationDispatcher dispatcher, ILoggerFactory loggerFactory)
: base(host, config, bindingProviders)
{
_dispatcher = dispatcher;
_loggerFactory = loggerFactory;
}
private async Task WaitForFunctionDispactherStateInitialized(IFunctionInvocationDispatcher functionDispatcher)
{
await TestHelpers.Await(() =>
{
return(functionDispatcher.State == FunctionInvocationDispatcherState.Initialized);
});
}
private async Task WaitForFunctionDispactherStateInitialized(IFunctionInvocationDispatcher functionDispatcher)
{
await TestHelpers.Await(() =>
{
return(functionDispatcher.State == FunctionInvocationDispatcherState.Initialized);
}, pollingInterval : 4 * 1000, timeout : 60 * 1000);
}
public WorkerFunctionDescriptorProvider(ScriptHost host, ScriptJobHostOptions config, ICollection <IScriptBindingProvider> bindingProviders,
IFunctionInvocationDispatcher dispatcher, ILoggerFactory loggerFactory, IApplicationLifetime applicationLifetime, TimeSpan workerInitializationTimeout)
: base(host, config, bindingProviders)
{
_dispatcher = dispatcher;
_loggerFactory = loggerFactory;
_applicationLifetime = applicationLifetime;
_workerInitializationTimeout = workerInitializationTimeout;
}
public AggregateFunctionMetadataProvider(
ILogger logger,
IFunctionInvocationDispatcher invocationDispatcher,
IFunctionMetadataProvider hostFunctionMetadataProvider,
IOptions <ScriptJobHostOptions> scriptOptions)
{
_logger = logger;
_dispatcher = invocationDispatcher;
_hostFunctionMetadataProvider = hostFunctionMetadataProvider;
_scriptOptions = scriptOptions;
}
internal WorkerFunctionInvoker(ScriptHost host, BindingMetadata bindingMetadata, FunctionMetadata functionMetadata, ILoggerFactory loggerFactory,
Collection <FunctionBinding> inputBindings, Collection <FunctionBinding> outputBindings, IFunctionInvocationDispatcher functionDispatcher)
: base(host, functionMetadata, loggerFactory)
{
_bindingMetadata = bindingMetadata;
_inputBindings = inputBindings;
_outputBindings = outputBindings;
_functionDispatcher = functionDispatcher;
_logger = loggerFactory.CreateLogger <WorkerFunctionInvoker>();
InitializeFileWatcherIfEnabled();
if (_outputBindings.Any(p => p.Metadata.IsReturn))
{
_handleScriptReturnValue = HandleReturnParameter;
}
else
{
_handleScriptReturnValue = HandleOutputDictionary;
}
}
public async Task OnTimeoutExceptionAsync(ExceptionDispatchInfo exceptionInfo, TimeSpan timeoutGracePeriod)
{
FunctionTimeoutException timeoutException = exceptionInfo.SourceException as FunctionTimeoutException;
if (timeoutException?.Task != null)
{
// We may double the timeoutGracePeriod here by first waiting to see if the iniital
// function task that started the exception has completed.
Task completedTask = await Task.WhenAny(timeoutException.Task, Task.Delay(timeoutGracePeriod));
// If the function task has completed, simply return. The host has already logged the timeout.
if (completedTask == timeoutException.Task)
{
return;
}
}
// We can't wait on this as it may cause a deadlock if the timeout was fired
// by a Listener that cannot stop until it has completed.
// TODO: DI (FACAVAL) The shutdown call will invoke the host stop... but we may need to do this
// explicitly in order to pass the timeout.
// Task ignoreTask = _hostManager.StopAsync();
// Give the manager and all running tasks some time to shut down gracefully.
//await Task.Delay(timeoutGracePeriod);
IFunctionInvocationDispatcher functionInvocationDispatcher = _functionInvocationDispatcherFactory.GetFunctionDispatcher();
if (!functionInvocationDispatcher.State.Equals(FunctionInvocationDispatcherState.Default))
{
_logger.LogWarning($"A function timeout has occurred. Restarting worker process executing invocationId '{timeoutException.InstanceId}'.", exceptionInfo.SourceException);
// If invocation id is not found in any of the workers => worker is already disposed. No action needed.
await functionInvocationDispatcher.RestartWorkerWithInvocationIdAsync(timeoutException.InstanceId.ToString());
_logger.LogWarning("Restart of language worker process(es) completed.", exceptionInfo.SourceException);
}
else
{
LogErrorAndFlush("A function timeout has occurred. Host is shutting down.", exceptionInfo.SourceException);
_applicationLifetime.StopApplication();
}
}
public FunctionInvocationDispatcherFactory(IOptions <ScriptJobHostOptions> scriptHostOptions,
IMetricsLogger metricsLogger,
IApplicationLifetime applicationLifetime,
IScriptEventManager eventManager,
ILoggerFactory loggerFactory,
IHttpWorkerChannelFactory httpWorkerChannelFactory,
IRpcWorkerChannelFactory rpcWorkerChannelFactory,
IOptions <HttpWorkerOptions> httpWorkerOptions,
IOptionsMonitor <LanguageWorkerOptions> rpcWorkerOptions,
IEnvironment environment,
IWebHostRpcWorkerChannelManager webHostLanguageWorkerChannelManager,
IJobHostRpcWorkerChannelManager jobHostLanguageWorkerChannelManager,
IOptions <ManagedDependencyOptions> managedDependencyOptions,
IRpcFunctionInvocationDispatcherLoadBalancer functionDispatcherLoadBalancer,
IOptions <WorkerConcurrencyOptions> workerConcurrencyOptions)
{
if (httpWorkerOptions.Value == null)
{
throw new ArgumentNullException(nameof(httpWorkerOptions.Value));
}
if (httpWorkerOptions.Value.Description != null)
{
_functionDispatcher = new HttpFunctionInvocationDispatcher(scriptHostOptions, metricsLogger, applicationLifetime, eventManager, loggerFactory, httpWorkerChannelFactory);
return;
}
_functionDispatcher = new RpcFunctionInvocationDispatcher(scriptHostOptions,
metricsLogger,
environment,
applicationLifetime,
eventManager,
loggerFactory,
rpcWorkerChannelFactory,
rpcWorkerOptions,
webHostLanguageWorkerChannelManager,
jobHostLanguageWorkerChannelManager,
managedDependencyOptions,
functionDispatcherLoadBalancer,
workerConcurrencyOptions);
}
public RpcFunctionDescriptorProvider(ScriptHost host, string workerRuntime, ScriptJobHostOptions config, ICollection <IScriptBindingProvider> bindingProviders,
IFunctionInvocationDispatcher dispatcher, ILoggerFactory loggerFactory, IApplicationLifetime applicationLifetime, TimeSpan workerInitializationTimeout)
: base(host, config, bindingProviders, dispatcher, loggerFactory, applicationLifetime, workerInitializationTimeout)
{
_workerRuntime = workerRuntime;
}
// Specify the "builtin binding types". These are types that are directly accesible without needing an explicit load gesture.
// This is the set of bindings we shipped prior to binding extensibility.
// Map from BindingType to the Assembly Qualified Type name for its IExtensionConfigProvider object.
public ScriptHost(IOptions <JobHostOptions> options,
IOptions <HttpWorkerOptions> httpWorkerOptions,
IEnvironment environment,
IJobHostContextFactory jobHostContextFactory,
IConfiguration configuration,
IDistributedLockManager distributedLockManager,
IScriptEventManager eventManager,
ILoggerFactory loggerFactory,
IFunctionInvocationDispatcherFactory functionDispatcherFactory,
IFunctionMetadataManager functionMetadataManager,
IFileLoggingStatusManager fileLoggingStatusManager,
IMetricsLogger metricsLogger,
IOptions <ScriptJobHostOptions> scriptHostOptions,
ITypeLocator typeLocator,
IScriptHostManager scriptHostManager,
IDebugStateProvider debugManager,
IEnumerable <IScriptBindingProvider> bindingProviders,
IPrimaryHostStateProvider primaryHostStateProvider,
IJobHostMetadataProvider metadataProvider,
IHostIdProvider hostIdProvider,
IHttpRoutesManager httpRoutesManager,
IApplicationLifetime applicationLifetime,
IExtensionBundleManager extensionBundleManager,
ScriptSettingsManager settingsManager = null)
: base(options, jobHostContextFactory)
{
_environment = environment;
_typeLocator = typeLocator as ScriptTypeLocator
?? throw new ArgumentException(nameof(typeLocator), $"A {nameof(ScriptTypeLocator)} instance is required.");
_instanceId = Guid.NewGuid().ToString();
_hostOptions = options;
_configuration = configuration;
_storageConnectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
_distributedLockManager = distributedLockManager;
_functionMetadataManager = functionMetadataManager;
_fileLoggingStatusManager = fileLoggingStatusManager;
_applicationLifetime = applicationLifetime;
_hostIdProvider = hostIdProvider;
_httpRoutesManager = httpRoutesManager;
_isHttpWorker = httpWorkerOptions.Value.Description != null;
ScriptOptions = scriptHostOptions.Value;
_scriptHostManager = scriptHostManager;
FunctionErrors = new Dictionary <string, ICollection <string> >(StringComparer.OrdinalIgnoreCase);
EventManager = eventManager;
_functionDispatcher = functionDispatcherFactory.GetFunctionDispatcher();
_settingsManager = settingsManager ?? ScriptSettingsManager.Instance;
ExtensionBundleManager = extensionBundleManager;
_metricsLogger = metricsLogger;
_hostLogPath = Path.Combine(ScriptOptions.RootLogPath, "Host");
_workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
_loggerFactory = loggerFactory;
_logger = loggerFactory.CreateLogger(LogCategories.Startup);
Logger = _logger;
_debugManager = debugManager;
_primaryHostStateProvider = primaryHostStateProvider;
_bindingProviders = new List <IScriptBindingProvider>(bindingProviders);
_metadataProvider = metadataProvider;
_eventSubscriptions.Add(EventManager.OfType <FunctionIndexingEvent>()
.Subscribe(evt =>
{
HandleHostError(evt.Exception);
}));
}
/// <summary>
/// Gets the function metadata array from all providers.
/// </summary>
/// <param name="forceRefresh">Forces reload from all providers.</param>
/// <param name="applyAllowList">Apply functions allow list filter.</param>
/// <param name="includeCustomProviders">Include any metadata provided by IFunctionProvider when loading the metadata</param>
/// <returns> An Immmutable array of FunctionMetadata.</returns>
public ImmutableArray <FunctionMetadata> GetFunctionMetadata(bool forceRefresh, bool applyAllowList = true, bool includeCustomProviders = true, IFunctionInvocationDispatcher dispatcher = null)
{
if (forceRefresh || _servicesReset || _functionMetadataArray.IsDefaultOrEmpty)
{
_functionMetadataArray = LoadFunctionMetadata(forceRefresh, includeCustomProviders, dispatcher);
_logger.FunctionMetadataManagerFunctionsLoaded(ApplyAllowList(_functionMetadataArray).Count());
_servicesReset = false;
}
return(applyAllowList ? ApplyAllowList(_functionMetadataArray) : _functionMetadataArray);
}
/// <summary>
/// Read all functions and populate function metadata.
/// </summary>
internal ImmutableArray <FunctionMetadata> LoadFunctionMetadata(bool forceRefresh = false, bool includeCustomProviders = true, IFunctionInvocationDispatcher dispatcher = null)
{
_functionMetadataMap.Clear();
ICollection <string> functionsAllowList = _scriptOptions?.Value?.Functions;
_logger.FunctionMetadataManagerLoadingFunctionsMetadata();
ImmutableArray <FunctionMetadata> immutableFunctionMetadata;
var workerConfigs = _languageWorkerOptions.Value.WorkerConfigs;
IFunctionMetadataProvider metadataProvider = new AggregateFunctionMetadataProvider(_loggerFactory.CreateLogger <AggregateFunctionMetadataProvider>(), dispatcher, _functionMetadataProvider, _scriptOptions);
immutableFunctionMetadata = metadataProvider.GetFunctionMetadataAsync(workerConfigs, SystemEnvironment.Instance, forceRefresh).GetAwaiter().GetResult();
var functionMetadataList = new List <FunctionMetadata>();
_functionErrors = new Dictionary <string, ICollection <string> >();
if (!immutableFunctionMetadata.IsDefaultOrEmpty)
{
functionMetadataList.AddRange(immutableFunctionMetadata);
}
if (!_functionMetadataProvider.FunctionErrors?.IsEmpty ?? false)
{
_functionErrors = _functionMetadataProvider.FunctionErrors.ToDictionary(kvp => kvp.Key, kvp => (ICollection <string>)kvp.Value.ToList());
}
// Add metadata and errors from any additional function providers
if (includeCustomProviders)
{
LoadCustomProviderFunctions(functionMetadataList);
}
// Validate
foreach (FunctionMetadata functionMetadata in functionMetadataList.ToList())
{
if (!IsScriptFileDetermined(functionMetadata))
{
// Exclude invalid functions
functionMetadataList.Remove(functionMetadata);
}
}
Errors = _functionErrors.ToImmutableDictionary(kvp => kvp.Key, kvp => kvp.Value.ToImmutableArray());
if (functionsAllowList != null)
{
_logger.LogInformation($"A function allow list has been specified, excluding all but the following functions: [{string.Join(", ", functionsAllowList)}]");
Errors = _functionErrors.Where(kvp => functionsAllowList.Any(functionName => functionName.Equals(kvp.Key, StringComparison.CurrentCultureIgnoreCase))).ToImmutableDictionary(kvp => kvp.Key, kvp => kvp.Value.ToImmutableArray());
}
return(functionMetadataList.OrderBy(f => f.Name, StringComparer.OrdinalIgnoreCase).ToImmutableArray());
}
public FunctionInvocationDispatcherShutdownManager(IFunctionInvocationDispatcherFactory functionDispatcherFactory)
{
_functionDispatcher = functionDispatcherFactory.GetFunctionDispatcher();
}
public ImmutableArray <FunctionMetadata> GetFunctionMetadata(bool forceRefresh = false, bool applyAllowlist = true, bool includeCustomProviders = true, IFunctionInvocationDispatcher dispatcher = null)
{
return(_functions.ToImmutableArray());
}
public HttpFunctionDescriptorProvider(ScriptHost host, ScriptJobHostOptions config, ICollection <IScriptBindingProvider> bindingProviders,
IFunctionInvocationDispatcher dispatcher, ILoggerFactory loggerFactory, IApplicationLifetime applicationLifetime)
: base(host, config, bindingProviders, dispatcher, loggerFactory, applicationLifetime)
{
}
public RpcFunctionDescriptorProvider(ScriptHost host, string workerRuntime, ScriptJobHostOptions config, ICollection <IScriptBindingProvider> bindingProviders,
IFunctionInvocationDispatcher dispatcher, ILoggerFactory loggerFactory)
: base(host, config, bindingProviders, dispatcher, loggerFactory)
{
_workerRuntime = workerRuntime;
}
public TestWorkerFunctionInvoker(ScriptHost host, BindingMetadata bindingMetadata, FunctionMetadata functionMetadata, ILoggerFactory loggerFactory,
Collection <FunctionBinding> inputBindings, Collection <FunctionBinding> outputBindings, IFunctionInvocationDispatcher functionDispatcher, IApplicationLifetime applicationLifetime,
TimeSpan initializationTimeout)
: base(host, bindingMetadata, functionMetadata, loggerFactory, inputBindings, outputBindings, functionDispatcher, applicationLifetime, initializationTimeout)
{
}
