public Task ShutdownChannelsAsync() { foreach (string runtime in _workerChannels.Keys) { _logger.LogInformation("Shutting down language worker channels for runtime:{runtime}", runtime); if (_workerChannels.TryRemove(runtime, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > standbyChannels)) { foreach (string workerId in standbyChannels.Keys) { standbyChannels[workerId]?.Task.ContinueWith(channelTask => { if (channelTask.Status == TaskStatus.Faulted) { _logger.LogDebug(channelTask.Exception, "Removing errored worker channel"); } else { IRpcWorkerChannel workerChannel = channelTask.Result; if (workerChannel != null) { (channelTask.Result as IDisposable)?.Dispose(); } } }); } } } return(Task.CompletedTask); }
public async Task SpecializeAsync() { _logger.LogInformation("Starting language worker channel specialization"); _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); IRpcWorkerChannel rpcWorkerChannel = await GetChannelAsync(_workerRuntime); if (_workerRuntime != null && rpcWorkerChannel != null) { if (UsePlaceholderChannel(_workerRuntime)) { _logger.LogDebug("Loading environment variables for runtime: {runtime}", _workerRuntime); await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest(); } else { _logger.LogDebug("Shutting down placeholder worker. Worker is not compatible for runtime: {runtime}", _workerRuntime); // If we need to allow file edits, we should shutdown the webhost channel on specialization. await ShutdownChannelIfExistsAsync(_workerRuntime, rpcWorkerChannel.Id); } } using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels)) { _shutdownStandbyWorkerChannels(); } _logger.LogDebug("Completed language worker channel specialization"); }
public async Task <IRpcWorkerChannel> InitializeChannelAsync(string language) { var metricsLogger = new Mock <IMetricsLogger>(); IRpcWorkerChannel workerChannel = _testLanguageWorkerChannelFactory.Create(_scriptRootPath, language, metricsLogger.Object, 0); if (_workerChannels.TryGetValue(language, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > workerChannels)) { workerChannels.Add(workerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>()); } else { _workerChannels.TryAdd(language, new Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> >()); _workerChannels[language].Add(workerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>()); } await workerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask => { if (processStartTask.Status == TaskStatus.RanToCompletion) { SetInitializedWorkerChannel(language, workerChannel); } else if (processStartTask.Status == TaskStatus.Faulted) { SetExceptionOnInitializedWorkerChannel(language, workerChannel, processStartTask.Exception); } }); return(workerChannel); }
public async Task <bool> ShutdownChannelIfExistsAsync(string language, string workerId) { if (string.IsNullOrEmpty(language)) { throw new ArgumentNullException(nameof(language)); } if (_workerChannels.TryGetValue(language, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > rpcWorkerChannels)) { if (rpcWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> value)) { try { IRpcWorkerChannel channel = await value?.Task; if (channel != null) { (channel as IDisposable)?.Dispose(); rpcWorkerChannels.Remove(workerId); return(true); } } catch (Exception) { rpcWorkerChannels.Remove(workerId); return(true); } } } return(false); }
public Task <bool> ShutdownChannelIfExistsAsync(string language, string workerId, Exception workerException = null) { if (string.IsNullOrEmpty(language)) { throw new ArgumentNullException(nameof(language)); } if (_workerChannels.TryRemove(language, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > rpcWorkerChannels)) { if (rpcWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> value)) { value?.Task.ContinueWith(channelTask => { if (channelTask.Status == TaskStatus.Faulted) { _logger.LogDebug(channelTask.Exception, "Removing errored worker channel"); } else { IRpcWorkerChannel workerChannel = channelTask.Result; if (workerChannel != null) { _logger.LogDebug("Disposing WebHost channel for workerId: {channelId}, for runtime:{language}", workerId, language); workerChannel.TryFailExecutions(workerException); (channelTask.Result as IDisposable)?.Dispose(); } } }); return(Task.FromResult(true)); } } return(Task.FromResult(false)); }
internal async Task <IRpcWorkerChannel> InitializeLanguageWorkerChannel(string runtime, string scriptRootPath) { IRpcWorkerChannel rpcWorkerChannel = null; string workerId = Guid.NewGuid().ToString(); _logger.LogDebug("Creating language worker channel for runtime:{runtime}", runtime); try { rpcWorkerChannel = _rpcWorkerChannelFactory.Create(scriptRootPath, runtime, _metricsLogger, 0, _lanuageworkerOptions.CurrentValue.WorkerConfigs); AddOrUpdateWorkerChannels(runtime, rpcWorkerChannel); await rpcWorkerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask => { if (processStartTask.Status == TaskStatus.RanToCompletion) { _logger.LogDebug("Adding jobhost language worker channel for runtime: {language}. workerId:{id}", _workerRuntime, rpcWorkerChannel.Id); SetInitializedWorkerChannel(runtime, rpcWorkerChannel); } else if (processStartTask.Status == TaskStatus.Faulted) { _logger.LogError("Failed to start language worker process for runtime: {language}. workerId:{id}", _workerRuntime, rpcWorkerChannel.Id); SetExceptionOnInitializedWorkerChannel(runtime, rpcWorkerChannel, processStartTask.Exception); } }); } catch (Exception ex) { throw new HostInitializationException($"Failed to start Language Worker Channel for language :{runtime}", ex); } return(rpcWorkerChannel); }
public async Task SpecializeAsync_LanguageWorkerArguments_KillsProcess(string languageWorkerName, string argument) { var testMetricsLogger = new TestMetricsLogger(); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, languageWorkerName); _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsExtensionVersion, "~3"); _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "1"); var config = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary <string, string> { [$"{RpcWorkerConstants.LanguageWorkersSectionName}:{languageWorkerName}:{WorkerConstants.WorkerDescriptionArguments}"] = argument }) .Build(); _rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger, _workerOptionsMonitor, config); IRpcWorkerChannel workerChannel = CreateTestChannel(languageWorkerName); await _rpcWorkerChannelManager.SpecializeAsync(); // Verify logs var traces = _testLogger.GetLogMessages(); Assert.True(traces.Count() == 0); // Verify channel var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(languageWorkerName); Assert.Null(initializedChannel); }
public async Task SpecializeAsync_ReadOnly_KeepsProcessAlive(string runtime, string languageWorkerName) { var testMetricsLogger = new TestMetricsLogger(); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, runtime); _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "1"); _rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger, _workerOptionsMonitor, _emptyConfig); IRpcWorkerChannel workerChannel = CreateTestChannel(languageWorkerName); await _rpcWorkerChannelManager.SpecializeAsync(); // Wait for debouce task to start await TestHelpers.Await(() => { return(testMetricsLogger.EventsBegan.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels) && testMetricsLogger.EventsEnded.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels)); }, pollingInterval : 500); // Verify logs var traces = _testLogger.GetLogMessages(); var functionLoadLogs = traces.Where(m => string.Equals(m.FormattedMessage, "SendFunctionEnvironmentReloadRequest called")); Assert.True(functionLoadLogs.Count() == 1); // Verify channel var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(languageWorkerName); Assert.Equal(workerChannel, initializedChannel); }
public async Task SpecializeAsync_Java_ReadOnly_KeepsProcessAlive() { var testMetricsLogger = new TestMetricsLogger(); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.JavaLanguageWorkerName); _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "1"); _rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger); IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); await _rpcWorkerChannelManager.SpecializeAsync(); Assert.True(testMetricsLogger.EventsBegan.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels) && testMetricsLogger.EventsEnded.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels)); // Verify logs var traces = _testLogger.GetLogMessages(); var functionLoadLogs = traces.Where(m => string.Equals(m.FormattedMessage, "SendFunctionEnvironmentReloadRequest called")); Assert.True(functionLoadLogs.Count() == 1); // Verify channel var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName); Assert.Equal(javaWorkerChannel, initializedChannel); }
public async Task ShutdownChannelsIfExistsAsync_StopsWorkerInvocations() { IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); Guid invocationId = Guid.NewGuid(); ScriptInvocationContext scriptInvocationContext = new ScriptInvocationContext() { ExecutionContext = new ExecutionContext() { InvocationId = invocationId } }; (javaWorkerChannel as TestRpcWorkerChannel).SendInvocationRequest(scriptInvocationContext); Assert.True(javaWorkerChannel.IsExecutingInvocation(invocationId.ToString())); Exception workerException = new Exception("Worker exception"); // Channel is removed immediately but is not failed immediately await _rpcWorkerChannelManager.ShutdownChannelIfExistsAsync(RpcWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel.Id, workerException); Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName)); var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName); Assert.Null(initializedChannel); // Execution will be terminated in the background - giving it 10 seconds await TestHelpers.Await(() => { return(!javaWorkerChannel.IsExecutingInvocation(invocationId.ToString())); }, 10000); Assert.False(javaWorkerChannel.IsExecutingInvocation(invocationId.ToString())); }
public void DisposeAndRemoveChannel(IRpcWorkerChannel channel) { if (_channels.TryRemove(channel.Id, out IRpcWorkerChannel removedChannel)) { _logger.LogDebug("Disposing language worker channel with id:{workerId}", removedChannel.Id); (removedChannel as IDisposable)?.Dispose(); } }
internal async void InitializeWebhostLanguageWorkerChannel() { _logger.LogDebug("Creating new webhost language worker channel for runtime:{workerRuntime}.", _workerRuntime); IRpcWorkerChannel workerChannel = await _webHostLanguageWorkerChannelManager.InitializeChannelAsync(_workerRuntime); workerChannel.SetupFunctionInvocationBuffers(_functions); workerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value); }
internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel) { if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > channel)) { if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource <IRpcWorkerChannel> value)) { value.SetResult(initializedLanguageWorkerChannel); } } }
public async Task ShutdownStandbyChannels_WorkerRuntime_Not_Set() { IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); await _rpcWorkerChannelManager.ShutdownChannelsAsync(); IRpcWorkerChannel initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName); Assert.Null(initializedChannel); }
internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel) { _logger.LogDebug("Adding webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id); if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > channel)) { if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource <IRpcWorkerChannel> value)) { value.SetResult(initializedLanguageWorkerChannel); } } }
public async Task InitializeAsync_WorkerRuntime_Node_DoNotInitialize_JavaWorker() { var channelManager = _rpcWorkerChannelManager as WebHostRpcWorkerChannelManager; IRpcWorkerChannel javaChannel = await channelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName); Assert.Null(javaChannel); IRpcWorkerChannel nodeChannel = await channelManager.GetChannelAsync(RpcWorkerConstants.NodeLanguageWorkerName); Assert.Null(nodeChannel); }
public async Task CreateChannels_Succeeds() { string language = RpcWorkerConstants.JavaLanguageWorkerName; IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(language); var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(language); IRpcWorkerChannel javaWorkerChannel2 = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); Assert.NotNull(initializedChannel); Assert.Equal(javaWorkerChannel.Id, initializedChannel.Id); Assert.Equal(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName).Count(), 2); }
internal async Task InitializeWebhostLanguageWorkerChannel() { _logger.LogDebug("Creating new webhost language worker channel for runtime:{workerRuntime}.", _workerRuntime); IRpcWorkerChannel workerChannel = await _webHostLanguageWorkerChannelManager.InitializeChannelAsync(_workerRuntime); // if the worker is indexing, we will not have function metadata yet so we cannot perform the next two lines if (!_workerIndexing) { workerChannel.SetupFunctionInvocationBuffers(_functions); workerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value, _scriptOptions.FunctionTimeout); SetFunctionDispatcherStateToInitializedAndLog(); } }
public async Task ShutdownChannelsIfExist_Succeeds() { IRpcWorkerChannel javaWorkerChannel1 = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); IRpcWorkerChannel javaWorkerChannel2 = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); await _rpcWorkerChannelManager.ShutdownChannelIfExistsAsync(RpcWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel1.Id); await _rpcWorkerChannelManager.ShutdownChannelIfExistsAsync(RpcWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel2.Id); Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName)); var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName); Assert.Null(initializedChannel); }
public async Task ShutdownStandyChannels_WorkerRuntime_Node_Set() { var testMetricsLogger = new TestMetricsLogger(); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.NodeLanguageWorkerName); _rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger); IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); _rpcWorkerChannelManager.ScheduleShutdownStandbyChannels(); Assert.True(AreRequiredMetricsEmitted(testMetricsLogger)); var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName); Assert.Null(initializedChannel); }
internal void AddOrUpdateWorkerChannels(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel) { _logger.LogDebug("Adding webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id); _workerChannels.AddOrUpdate(initializedRuntime, (runtime) => { Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > newLanguageWorkerChannels = new Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> >(); newLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>()); return(newLanguageWorkerChannels); }, (runtime, existingLanguageWorkerChannels) => { existingLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>()); return(existingLanguageWorkerChannels); }); }
internal void ScheduleShutdownStandbyChannels() { using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels)) { _workerRuntime = _workerRuntime ?? _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); if (!string.IsNullOrEmpty(_workerRuntime)) { var standbyWorkerChannels = _workerChannels.Where(ch => !ch.Key.Equals(_workerRuntime, StringComparison.InvariantCultureIgnoreCase)); foreach (var runtime in standbyWorkerChannels) { using (_metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannels, runtime.Key))) { _logger.LogInformation("Disposing standby channel for runtime:{language}", runtime.Key); if (_workerChannels.TryRemove(runtime.Key, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > standbyChannels)) { foreach (string workerId in standbyChannels.Keys) { IDisposable latencyEvent = _metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannel, workerId)); standbyChannels[workerId]?.Task.ContinueWith(channelTask => { if (channelTask.Status == TaskStatus.Faulted) { _logger.LogDebug(channelTask.Exception, "Removing errored worker channel"); } else { IRpcWorkerChannel workerChannel = channelTask.Result; if (workerChannel != null) { (channelTask.Result as IDisposable)?.Dispose(); } } latencyEvent.Dispose(); }); } } } } } } }
public async Task ShutdownChannels_Succeeds() { string javaWorkerId = Guid.NewGuid().ToString(); IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName); string nodeWorkerId = Guid.NewGuid().ToString(); IRpcWorkerChannel nodeWorkerChannel = CreateTestChannel(RpcWorkerConstants.NodeLanguageWorkerName); // Shutdown await _rpcWorkerChannelManager.ShutdownChannelsAsync(); Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName)); Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.NodeLanguageWorkerName)); // Verify disposed IRpcWorkerChannel initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.NodeLanguageWorkerName); Assert.Null(initializedChannel); initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName); Assert.Null(initializedChannel); }
public async Task SpecializeAsync_Node_V2CompatibilityWithV3Extension_KillsProcess(string languageWorkerName) { var testMetricsLogger = new TestMetricsLogger(); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, languageWorkerName); _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsV2CompatibilityModeKey, "true"); _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsExtensionVersion, "~3"); _rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger, _workerOptionsMonitor); IRpcWorkerChannel workerChannel = CreateTestChannel(languageWorkerName); await _rpcWorkerChannelManager.SpecializeAsync(); // Verify logs var traces = _testLogger.GetLogMessages(); Assert.True(traces.Count() == 0); // Verify channel var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(languageWorkerName); Assert.Null(initializedChannel); }
public async Task SpecializeAsync_Node_NotReadOnly_KillsProcess() { var testMetricsLogger = new TestMetricsLogger(); _testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.NodeLanguageWorkerName); // This is an invalid setting configuration, but just to show that run from zip is NOT set _testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "0"); _rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger); IRpcWorkerChannel nodeWorkerChannel = CreateTestChannel(RpcWorkerConstants.NodeLanguageWorkerName); await _rpcWorkerChannelManager.SpecializeAsync(); // Verify logs var traces = _testLogger.GetLogMessages(); Assert.True(traces.Count() == 0); // Verify channel var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.NodeLanguageWorkerName); Assert.Null(initializedChannel); }
public async Task InitializeAsync(IEnumerable <FunctionMetadata> functions, CancellationToken cancellationToken = default) { if (_environment.IsPlaceholderModeEnabled()) { return; } _workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions); _functions = functions; if (string.IsNullOrEmpty(_workerRuntime) || _workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.InvariantCultureIgnoreCase)) { // Shutdown any placeholder channels for empty function apps or dotnet function apps. // This is needed as specilization does not kill standby placeholder channels if worker runtime is not set. // Debouce to ensure this does not effect cold start _shutdownStandbyWorkerChannels(); return; } if (functions == null || functions.Count() == 0) { // do not initialize function dispachter if there are no functions return; } if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs)) { State = FunctionInvocationDispatcherState.Initializing; Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime); if (webhostLanguageWorkerChannels != null) { foreach (string workerId in webhostLanguageWorkerChannels.Keys.ToList()) { if (webhostLanguageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> initializedLanguageWorkerChannelTask)) { _logger.LogDebug("Found initialized language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId); try { IRpcWorkerChannel initializedLanguageWorkerChannel = await initializedLanguageWorkerChannelTask.Task; initializedLanguageWorkerChannel.SetupFunctionInvocationBuffers(_functions); initializedLanguageWorkerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value); } catch (Exception ex) { _logger.LogWarning(ex, "Removing errored webhost language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId); await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(_workerRuntime, workerId); InitializeWebhostLanguageWorkerChannel(); } } } StartWorkerProcesses(webhostLanguageWorkerChannels.Count(), InitializeWebhostLanguageWorkerChannel); State = FunctionInvocationDispatcherState.Initialized; } else { await InitializeJobhostLanguageWorkerChannelAsync(0); StartWorkerProcesses(1, InitializeJobhostLanguageWorkerChannelAsync); } } }
public void AddChannel(IRpcWorkerChannel channel) { _channels.TryAdd(channel.Id, channel); }
public async Task InitializeAsync(IEnumerable <FunctionMetadata> functions, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); var placeholderModeEnabled = _environment.IsPlaceholderModeEnabled(); _logger.LogDebug($"Placeholder mode is enabled: {placeholderModeEnabled}"); if (placeholderModeEnabled) { return; } _workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions, _environment); if (string.IsNullOrEmpty(_workerRuntime) || _workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.InvariantCultureIgnoreCase)) { // Shutdown any placeholder channels for empty function apps or dotnet function apps. // This is needed as specilization does not kill standby placeholder channels if worker runtime is not set. // Debouce to ensure this does not effect cold start _shutdownStandbyWorkerChannels(); return; } var workerConfig = _workerConfigs.Where(c => c.Description.Language.Equals(_workerRuntime, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault(); if (workerConfig == null && (functions == null || functions.Count() == 0)) { // Only throw if workerConfig is null AND some functions have been found. // With .NET out-of-proc, worker config comes from functions. var allLanguageNamesFromWorkerConfigs = string.Join(",", _workerConfigs.Select(c => c.Description.Language)); _logger.LogDebug($"Languages present in WorkerConfig: {allLanguageNamesFromWorkerConfigs}"); throw new InvalidOperationException($"WorkerConfig for runtime: {_workerRuntime} not found"); } if ((functions == null || functions.Count() == 0) && !_workerIndexing) { // do not initialize function dispatcher if there are no functions, unless the worker is indexing _logger.LogDebug($"{nameof(RpcFunctionInvocationDispatcher)} received no functions"); return; } _functions = functions ?? new List <FunctionMetadata>(); _processStartupInterval = workerConfig.CountOptions.ProcessStartupInterval; _restartWait = workerConfig.CountOptions.ProcessRestartInterval; _shutdownTimeout = workerConfig.CountOptions.ProcessShutdownTimeout; ErrorEventsThreshold = 3 * (await _maxProcessCount.Value); if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs)) { State = FunctionInvocationDispatcherState.Initializing; Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime); if (webhostLanguageWorkerChannels != null) { int countOfReadyChannels = 0; foreach (string workerId in webhostLanguageWorkerChannels.Keys.ToList()) { if (webhostLanguageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> initializedLanguageWorkerChannelTask)) { _logger.LogDebug("Found initialized language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId); try { IRpcWorkerChannel initializedLanguageWorkerChannel = await initializedLanguageWorkerChannelTask.Task; // if worker is not indexing, then _functions is populated and we can set up invocation buffers and send load requests if (!_workerIndexing) { initializedLanguageWorkerChannel.SetupFunctionInvocationBuffers(_functions); initializedLanguageWorkerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value, _scriptOptions.FunctionTimeout); ++countOfReadyChannels; } } catch (Exception ex) { _logger.LogWarning(ex, "Removing errored webhost language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId); await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(_workerRuntime, workerId, ex); } } } StartWorkerProcesses(countOfReadyChannels, InitializeWebhostLanguageWorkerChannel, true, _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime)); } else { // if _workerIndexing, initialize a single channel and let the rest start up the background if (_workerIndexing) { await InitializeJobhostLanguageWorkerChannelAsync(); StartWorkerProcesses(1, InitializeJobhostLanguageWorkerChannelAsync); } else { StartWorkerProcesses(0, InitializeJobhostLanguageWorkerChannelAsync); } } } }
public async Task InitializeAsync(IEnumerable <FunctionMetadata> functions, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); if (_environment.IsPlaceholderModeEnabled()) { return; } _workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions); _functions = functions; if (string.IsNullOrEmpty(_workerRuntime) || _workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.InvariantCultureIgnoreCase)) { // Shutdown any placeholder channels for empty function apps or dotnet function apps. // This is needed as specilization does not kill standby placeholder channels if worker runtime is not set. // Debouce to ensure this does not effect cold start _shutdownStandbyWorkerChannels(); return; } if (functions == null || functions.Count() == 0) { // do not initialize function dispatcher if there are no functions return; } var workerConfig = _workerConfigs.Where(c => c.Description.Language.Equals(_workerRuntime, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); if (workerConfig == null) { // Only throw if workerConfig is null AND some functions have been found. // With .NET out-of-proc, worker config comes from functions. throw new InvalidOperationException($"WorkerCofig for runtime: {_workerRuntime} not found"); } _maxProcessCount = workerConfig.CountOptions.ProcessCount; _debounceMilliSeconds = (int)workerConfig.CountOptions.ProcessStartupInterval.TotalMilliseconds; ErrorEventsThreshold = 3 * _maxProcessCount; if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs)) { State = FunctionInvocationDispatcherState.Initializing; Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime); if (webhostLanguageWorkerChannels != null) { foreach (string workerId in webhostLanguageWorkerChannels.Keys.ToList()) { if (webhostLanguageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> initializedLanguageWorkerChannelTask)) { _logger.LogDebug("Found initialized language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId); try { IRpcWorkerChannel initializedLanguageWorkerChannel = await initializedLanguageWorkerChannelTask.Task; initializedLanguageWorkerChannel.SetupFunctionInvocationBuffers(_functions); initializedLanguageWorkerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value, _scriptOptions.FunctionTimeout); } catch (Exception ex) { _logger.LogWarning(ex, "Removing errored webhost language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId); await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(_workerRuntime, workerId, ex); InitializeWebhostLanguageWorkerChannel(); } } } StartWorkerProcesses(webhostLanguageWorkerChannels.Count(), InitializeWebhostLanguageWorkerChannel); SetFunctionDispatcherStateToInitializedAndLog(); } else { await InitializeJobhostLanguageWorkerChannelAsync(0); StartWorkerProcesses(1, InitializeJobhostLanguageWorkerChannelAsync); } } }