public Task ShutdownChannelsAsync()
{
foreach (string runtime in _workerChannels.Keys)
{
_logger.LogInformation("Shutting down language worker channels for runtime:{runtime}", runtime);
if (_workerChannels.TryRemove(runtime, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > standbyChannels))
{
foreach (string workerId in standbyChannels.Keys)
{
standbyChannels[workerId]?.Task.ContinueWith(channelTask =>
{
if (channelTask.Status == TaskStatus.Faulted)
{
_logger.LogDebug(channelTask.Exception, "Removing errored worker channel");
}
else
{
IRpcWorkerChannel workerChannel = channelTask.Result;
if (workerChannel != null)
{
(channelTask.Result as IDisposable)?.Dispose();
}
}
});
}
}
}
return(Task.CompletedTask);
}
public async Task SpecializeAsync()
{
_logger.LogInformation("Starting language worker channel specialization");
_workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
IRpcWorkerChannel rpcWorkerChannel = await GetChannelAsync(_workerRuntime);
if (_workerRuntime != null && rpcWorkerChannel != null)
{
if (UsePlaceholderChannel(_workerRuntime))
{
_logger.LogDebug("Loading environment variables for runtime: {runtime}", _workerRuntime);
await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest();
}
else
{
_logger.LogDebug("Shutting down placeholder worker. Worker is not compatible for runtime: {runtime}", _workerRuntime);
// If we need to allow file edits, we should shutdown the webhost channel on specialization.
await ShutdownChannelIfExistsAsync(_workerRuntime, rpcWorkerChannel.Id);
}
}
using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels))
{
_shutdownStandbyWorkerChannels();
}
_logger.LogDebug("Completed language worker channel specialization");
}
public async Task <IRpcWorkerChannel> InitializeChannelAsync(string language)
{
var metricsLogger = new Mock <IMetricsLogger>();
IRpcWorkerChannel workerChannel = _testLanguageWorkerChannelFactory.Create(_scriptRootPath, language, metricsLogger.Object, 0);
if (_workerChannels.TryGetValue(language, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > workerChannels))
{
workerChannels.Add(workerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>());
}
else
{
_workerChannels.TryAdd(language, new Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> >());
_workerChannels[language].Add(workerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>());
}
await workerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask =>
{
if (processStartTask.Status == TaskStatus.RanToCompletion)
{
SetInitializedWorkerChannel(language, workerChannel);
}
else if (processStartTask.Status == TaskStatus.Faulted)
{
SetExceptionOnInitializedWorkerChannel(language, workerChannel, processStartTask.Exception);
}
});
return(workerChannel);
}
public async Task <bool> ShutdownChannelIfExistsAsync(string language, string workerId)
{
if (string.IsNullOrEmpty(language))
{
throw new ArgumentNullException(nameof(language));
}
if (_workerChannels.TryGetValue(language, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > rpcWorkerChannels))
{
if (rpcWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> value))
{
try
{
IRpcWorkerChannel channel = await value?.Task;
if (channel != null)
{
(channel as IDisposable)?.Dispose();
rpcWorkerChannels.Remove(workerId);
return(true);
}
}
catch (Exception)
{
rpcWorkerChannels.Remove(workerId);
return(true);
}
}
}
return(false);
}
public Task <bool> ShutdownChannelIfExistsAsync(string language, string workerId, Exception workerException = null)
{
if (string.IsNullOrEmpty(language))
{
throw new ArgumentNullException(nameof(language));
}
if (_workerChannels.TryRemove(language, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > rpcWorkerChannels))
{
if (rpcWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> value))
{
value?.Task.ContinueWith(channelTask =>
{
if (channelTask.Status == TaskStatus.Faulted)
{
_logger.LogDebug(channelTask.Exception, "Removing errored worker channel");
}
else
{
IRpcWorkerChannel workerChannel = channelTask.Result;
if (workerChannel != null)
{
_logger.LogDebug("Disposing WebHost channel for workerId: {channelId}, for runtime:{language}", workerId, language);
workerChannel.TryFailExecutions(workerException);
(channelTask.Result as IDisposable)?.Dispose();
}
}
});
return(Task.FromResult(true));
}
}
return(Task.FromResult(false));
}
internal async Task <IRpcWorkerChannel> InitializeLanguageWorkerChannel(string runtime, string scriptRootPath)
{
IRpcWorkerChannel rpcWorkerChannel = null;
string workerId = Guid.NewGuid().ToString();
_logger.LogDebug("Creating language worker channel for runtime:{runtime}", runtime);
try
{
rpcWorkerChannel = _rpcWorkerChannelFactory.Create(scriptRootPath, runtime, _metricsLogger, 0, _lanuageworkerOptions.CurrentValue.WorkerConfigs);
AddOrUpdateWorkerChannels(runtime, rpcWorkerChannel);
await rpcWorkerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask =>
{
if (processStartTask.Status == TaskStatus.RanToCompletion)
{
_logger.LogDebug("Adding jobhost language worker channel for runtime: {language}. workerId:{id}", _workerRuntime, rpcWorkerChannel.Id);
SetInitializedWorkerChannel(runtime, rpcWorkerChannel);
}
else if (processStartTask.Status == TaskStatus.Faulted)
{
_logger.LogError("Failed to start language worker process for runtime: {language}. workerId:{id}", _workerRuntime, rpcWorkerChannel.Id);
SetExceptionOnInitializedWorkerChannel(runtime, rpcWorkerChannel, processStartTask.Exception);
}
});
}
catch (Exception ex)
{
throw new HostInitializationException($"Failed to start Language Worker Channel for language :{runtime}", ex);
}
return(rpcWorkerChannel);
}
public async Task SpecializeAsync_LanguageWorkerArguments_KillsProcess(string languageWorkerName, string argument)
{
var testMetricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, languageWorkerName);
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsExtensionVersion, "~3");
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "1");
var config = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary <string, string>
{
[$"{RpcWorkerConstants.LanguageWorkersSectionName}:{languageWorkerName}:{WorkerConstants.WorkerDescriptionArguments}"] = argument
})
.Build();
_rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger, _workerOptionsMonitor, config);
IRpcWorkerChannel workerChannel = CreateTestChannel(languageWorkerName);
await _rpcWorkerChannelManager.SpecializeAsync();
// Verify logs
var traces = _testLogger.GetLogMessages();
Assert.True(traces.Count() == 0);
// Verify channel
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(languageWorkerName);
Assert.Null(initializedChannel);
}
public async Task SpecializeAsync_ReadOnly_KeepsProcessAlive(string runtime, string languageWorkerName)
{
var testMetricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, runtime);
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "1");
_rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger, _workerOptionsMonitor, _emptyConfig);
IRpcWorkerChannel workerChannel = CreateTestChannel(languageWorkerName);
await _rpcWorkerChannelManager.SpecializeAsync();
// Wait for debouce task to start
await TestHelpers.Await(() =>
{
return(testMetricsLogger.EventsBegan.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels) &&
testMetricsLogger.EventsEnded.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels));
}, pollingInterval : 500);
// Verify logs
var traces = _testLogger.GetLogMessages();
var functionLoadLogs = traces.Where(m => string.Equals(m.FormattedMessage, "SendFunctionEnvironmentReloadRequest called"));
Assert.True(functionLoadLogs.Count() == 1);
// Verify channel
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(languageWorkerName);
Assert.Equal(workerChannel, initializedChannel);
}
public async Task SpecializeAsync_Java_ReadOnly_KeepsProcessAlive()
{
var testMetricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.JavaLanguageWorkerName);
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "1");
_rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger);
IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
await _rpcWorkerChannelManager.SpecializeAsync();
Assert.True(testMetricsLogger.EventsBegan.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels) &&
testMetricsLogger.EventsEnded.Contains(MetricEventNames.SpecializationScheduleShutdownStandbyChannels));
// Verify logs
var traces = _testLogger.GetLogMessages();
var functionLoadLogs = traces.Where(m => string.Equals(m.FormattedMessage, "SendFunctionEnvironmentReloadRequest called"));
Assert.True(functionLoadLogs.Count() == 1);
// Verify channel
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Equal(javaWorkerChannel, initializedChannel);
}
public async Task ShutdownChannelsIfExistsAsync_StopsWorkerInvocations()
{
IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
Guid invocationId = Guid.NewGuid();
ScriptInvocationContext scriptInvocationContext = new ScriptInvocationContext()
{
ExecutionContext = new ExecutionContext()
{
InvocationId = invocationId
}
};
(javaWorkerChannel as TestRpcWorkerChannel).SendInvocationRequest(scriptInvocationContext);
Assert.True(javaWorkerChannel.IsExecutingInvocation(invocationId.ToString()));
Exception workerException = new Exception("Worker exception");
// Channel is removed immediately but is not failed immediately
await _rpcWorkerChannelManager.ShutdownChannelIfExistsAsync(RpcWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel.Id, workerException);
Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName));
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
// Execution will be terminated in the background - giving it 10 seconds
await TestHelpers.Await(() =>
{
return(!javaWorkerChannel.IsExecutingInvocation(invocationId.ToString()));
}, 10000);
Assert.False(javaWorkerChannel.IsExecutingInvocation(invocationId.ToString()));
}
public void DisposeAndRemoveChannel(IRpcWorkerChannel channel)
{
if (_channels.TryRemove(channel.Id, out IRpcWorkerChannel removedChannel))
{
_logger.LogDebug("Disposing language worker channel with id:{workerId}", removedChannel.Id);
(removedChannel as IDisposable)?.Dispose();
}
}
internal async void InitializeWebhostLanguageWorkerChannel()
{
_logger.LogDebug("Creating new webhost language worker channel for runtime:{workerRuntime}.", _workerRuntime);
IRpcWorkerChannel workerChannel = await _webHostLanguageWorkerChannelManager.InitializeChannelAsync(_workerRuntime);
workerChannel.SetupFunctionInvocationBuffers(_functions);
workerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value);
}
internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel)
{
if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > channel))
{
if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource <IRpcWorkerChannel> value))
{
value.SetResult(initializedLanguageWorkerChannel);
}
}
}
public async Task ShutdownStandbyChannels_WorkerRuntime_Not_Set()
{
IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
await _rpcWorkerChannelManager.ShutdownChannelsAsync();
IRpcWorkerChannel initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
internal void SetInitializedWorkerChannel(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel)
{
_logger.LogDebug("Adding webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id);
if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > channel))
{
if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource <IRpcWorkerChannel> value))
{
value.SetResult(initializedLanguageWorkerChannel);
}
}
}
public async Task InitializeAsync_WorkerRuntime_Node_DoNotInitialize_JavaWorker()
{
var channelManager = _rpcWorkerChannelManager as WebHostRpcWorkerChannelManager;
IRpcWorkerChannel javaChannel = await channelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Null(javaChannel);
IRpcWorkerChannel nodeChannel = await channelManager.GetChannelAsync(RpcWorkerConstants.NodeLanguageWorkerName);
Assert.Null(nodeChannel);
}
public async Task CreateChannels_Succeeds()
{
string language = RpcWorkerConstants.JavaLanguageWorkerName;
IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(language);
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(language);
IRpcWorkerChannel javaWorkerChannel2 = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.NotNull(initializedChannel);
Assert.Equal(javaWorkerChannel.Id, initializedChannel.Id);
Assert.Equal(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName).Count(), 2);
}
internal async Task InitializeWebhostLanguageWorkerChannel()
{
_logger.LogDebug("Creating new webhost language worker channel for runtime:{workerRuntime}.", _workerRuntime);
IRpcWorkerChannel workerChannel = await _webHostLanguageWorkerChannelManager.InitializeChannelAsync(_workerRuntime);
// if the worker is indexing, we will not have function metadata yet so we cannot perform the next two lines
if (!_workerIndexing)
{
workerChannel.SetupFunctionInvocationBuffers(_functions);
workerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value, _scriptOptions.FunctionTimeout);
SetFunctionDispatcherStateToInitializedAndLog();
}
}
public async Task ShutdownChannelsIfExist_Succeeds()
{
IRpcWorkerChannel javaWorkerChannel1 = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
IRpcWorkerChannel javaWorkerChannel2 = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
await _rpcWorkerChannelManager.ShutdownChannelIfExistsAsync(RpcWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel1.Id);
await _rpcWorkerChannelManager.ShutdownChannelIfExistsAsync(RpcWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel2.Id);
Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName));
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public async Task ShutdownStandyChannels_WorkerRuntime_Node_Set()
{
var testMetricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.NodeLanguageWorkerName);
_rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger);
IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
_rpcWorkerChannelManager.ScheduleShutdownStandbyChannels();
Assert.True(AreRequiredMetricsEmitted(testMetricsLogger));
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
internal void AddOrUpdateWorkerChannels(string initializedRuntime, IRpcWorkerChannel initializedLanguageWorkerChannel)
{
_logger.LogDebug("Adding webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id);
_workerChannels.AddOrUpdate(initializedRuntime,
(runtime) =>
{
Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > newLanguageWorkerChannels = new Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> >();
newLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>());
return(newLanguageWorkerChannels);
},
(runtime, existingLanguageWorkerChannels) =>
{
existingLanguageWorkerChannels.Add(initializedLanguageWorkerChannel.Id, new TaskCompletionSource <IRpcWorkerChannel>());
return(existingLanguageWorkerChannels);
});
}
internal void ScheduleShutdownStandbyChannels()
{
using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels))
{
_workerRuntime = _workerRuntime ?? _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
if (!string.IsNullOrEmpty(_workerRuntime))
{
var standbyWorkerChannels = _workerChannels.Where(ch => !ch.Key.Equals(_workerRuntime, StringComparison.InvariantCultureIgnoreCase));
foreach (var runtime in standbyWorkerChannels)
{
using (_metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannels, runtime.Key)))
{
_logger.LogInformation("Disposing standby channel for runtime:{language}", runtime.Key);
if (_workerChannels.TryRemove(runtime.Key, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > standbyChannels))
{
foreach (string workerId in standbyChannels.Keys)
{
IDisposable latencyEvent = _metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannel, workerId));
standbyChannels[workerId]?.Task.ContinueWith(channelTask =>
{
if (channelTask.Status == TaskStatus.Faulted)
{
_logger.LogDebug(channelTask.Exception, "Removing errored worker channel");
}
else
{
IRpcWorkerChannel workerChannel = channelTask.Result;
if (workerChannel != null)
{
(channelTask.Result as IDisposable)?.Dispose();
}
}
latencyEvent.Dispose();
});
}
}
}
}
}
}
}
public async Task ShutdownChannels_Succeeds()
{
string javaWorkerId = Guid.NewGuid().ToString();
IRpcWorkerChannel javaWorkerChannel = CreateTestChannel(RpcWorkerConstants.JavaLanguageWorkerName);
string nodeWorkerId = Guid.NewGuid().ToString();
IRpcWorkerChannel nodeWorkerChannel = CreateTestChannel(RpcWorkerConstants.NodeLanguageWorkerName);
// Shutdown
await _rpcWorkerChannelManager.ShutdownChannelsAsync();
Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.JavaLanguageWorkerName));
Assert.Null(_rpcWorkerChannelManager.GetChannels(RpcWorkerConstants.NodeLanguageWorkerName));
// Verify disposed
IRpcWorkerChannel initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.NodeLanguageWorkerName);
Assert.Null(initializedChannel);
initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public async Task SpecializeAsync_Node_V2CompatibilityWithV3Extension_KillsProcess(string languageWorkerName)
{
var testMetricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, languageWorkerName);
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsV2CompatibilityModeKey, "true");
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsExtensionVersion, "~3");
_rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger, _workerOptionsMonitor);
IRpcWorkerChannel workerChannel = CreateTestChannel(languageWorkerName);
await _rpcWorkerChannelManager.SpecializeAsync();
// Verify logs
var traces = _testLogger.GetLogMessages();
Assert.True(traces.Count() == 0);
// Verify channel
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(languageWorkerName);
Assert.Null(initializedChannel);
}
public async Task SpecializeAsync_Node_NotReadOnly_KillsProcess()
{
var testMetricsLogger = new TestMetricsLogger();
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, RpcWorkerConstants.NodeLanguageWorkerName);
// This is an invalid setting configuration, but just to show that run from zip is NOT set
_testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteZipDeployment, "0");
_rpcWorkerChannelManager = new WebHostRpcWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _rpcWorkerChannelFactory, _optionsMonitor, testMetricsLogger);
IRpcWorkerChannel nodeWorkerChannel = CreateTestChannel(RpcWorkerConstants.NodeLanguageWorkerName);
await _rpcWorkerChannelManager.SpecializeAsync();
// Verify logs
var traces = _testLogger.GetLogMessages();
Assert.True(traces.Count() == 0);
// Verify channel
var initializedChannel = await _rpcWorkerChannelManager.GetChannelAsync(RpcWorkerConstants.NodeLanguageWorkerName);
Assert.Null(initializedChannel);
}
public async Task InitializeAsync(IEnumerable <FunctionMetadata> functions, CancellationToken cancellationToken = default)
{
if (_environment.IsPlaceholderModeEnabled())
{
return;
}
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions);
_functions = functions;
if (string.IsNullOrEmpty(_workerRuntime) || _workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.InvariantCultureIgnoreCase))
{
// Shutdown any placeholder channels for empty function apps or dotnet function apps.
// This is needed as specilization does not kill standby placeholder channels if worker runtime is not set.
// Debouce to ensure this does not effect cold start
_shutdownStandbyWorkerChannels();
return;
}
if (functions == null || functions.Count() == 0)
{
// do not initialize function dispachter if there are no functions
return;
}
if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs))
{
State = FunctionInvocationDispatcherState.Initializing;
Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);
if (webhostLanguageWorkerChannels != null)
{
foreach (string workerId in webhostLanguageWorkerChannels.Keys.ToList())
{
if (webhostLanguageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> initializedLanguageWorkerChannelTask))
{
_logger.LogDebug("Found initialized language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
try
{
IRpcWorkerChannel initializedLanguageWorkerChannel = await initializedLanguageWorkerChannelTask.Task;
initializedLanguageWorkerChannel.SetupFunctionInvocationBuffers(_functions);
initializedLanguageWorkerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Removing errored webhost language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(_workerRuntime, workerId);
InitializeWebhostLanguageWorkerChannel();
}
}
}
StartWorkerProcesses(webhostLanguageWorkerChannels.Count(), InitializeWebhostLanguageWorkerChannel);
State = FunctionInvocationDispatcherState.Initialized;
}
else
{
await InitializeJobhostLanguageWorkerChannelAsync(0);
StartWorkerProcesses(1, InitializeJobhostLanguageWorkerChannelAsync);
}
}
}
public void AddChannel(IRpcWorkerChannel channel)
{
_channels.TryAdd(channel.Id, channel);
}
public async Task InitializeAsync(IEnumerable <FunctionMetadata> functions, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var placeholderModeEnabled = _environment.IsPlaceholderModeEnabled();
_logger.LogDebug($"Placeholder mode is enabled: {placeholderModeEnabled}");
if (placeholderModeEnabled)
{
return;
}
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions, _environment);
if (string.IsNullOrEmpty(_workerRuntime) || _workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.InvariantCultureIgnoreCase))
{
// Shutdown any placeholder channels for empty function apps or dotnet function apps.
// This is needed as specilization does not kill standby placeholder channels if worker runtime is not set.
// Debouce to ensure this does not effect cold start
_shutdownStandbyWorkerChannels();
return;
}
var workerConfig = _workerConfigs.Where(c => c.Description.Language.Equals(_workerRuntime, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
if (workerConfig == null && (functions == null || functions.Count() == 0))
{
// Only throw if workerConfig is null AND some functions have been found.
// With .NET out-of-proc, worker config comes from functions.
var allLanguageNamesFromWorkerConfigs = string.Join(",", _workerConfigs.Select(c => c.Description.Language));
_logger.LogDebug($"Languages present in WorkerConfig: {allLanguageNamesFromWorkerConfigs}");
throw new InvalidOperationException($"WorkerConfig for runtime: {_workerRuntime} not found");
}
if ((functions == null || functions.Count() == 0) && !_workerIndexing)
{
// do not initialize function dispatcher if there are no functions, unless the worker is indexing
_logger.LogDebug($"{nameof(RpcFunctionInvocationDispatcher)} received no functions");
return;
}
_functions = functions ?? new List <FunctionMetadata>();
_processStartupInterval = workerConfig.CountOptions.ProcessStartupInterval;
_restartWait = workerConfig.CountOptions.ProcessRestartInterval;
_shutdownTimeout = workerConfig.CountOptions.ProcessShutdownTimeout;
ErrorEventsThreshold = 3 * (await _maxProcessCount.Value);
if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs))
{
State = FunctionInvocationDispatcherState.Initializing;
Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);
if (webhostLanguageWorkerChannels != null)
{
int countOfReadyChannels = 0;
foreach (string workerId in webhostLanguageWorkerChannels.Keys.ToList())
{
if (webhostLanguageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> initializedLanguageWorkerChannelTask))
{
_logger.LogDebug("Found initialized language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
try
{
IRpcWorkerChannel initializedLanguageWorkerChannel = await initializedLanguageWorkerChannelTask.Task;
// if worker is not indexing, then _functions is populated and we can set up invocation buffers and send load requests
if (!_workerIndexing)
{
initializedLanguageWorkerChannel.SetupFunctionInvocationBuffers(_functions);
initializedLanguageWorkerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value, _scriptOptions.FunctionTimeout);
++countOfReadyChannels;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Removing errored webhost language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(_workerRuntime, workerId, ex);
}
}
}
StartWorkerProcesses(countOfReadyChannels, InitializeWebhostLanguageWorkerChannel, true, _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime));
}
else
{
// if _workerIndexing, initialize a single channel and let the rest start up the background
if (_workerIndexing)
{
await InitializeJobhostLanguageWorkerChannelAsync();
StartWorkerProcesses(1, InitializeJobhostLanguageWorkerChannelAsync);
}
else
{
StartWorkerProcesses(0, InitializeJobhostLanguageWorkerChannelAsync);
}
}
}
}
public async Task InitializeAsync(IEnumerable <FunctionMetadata> functions, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
if (_environment.IsPlaceholderModeEnabled())
{
return;
}
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions);
_functions = functions;
if (string.IsNullOrEmpty(_workerRuntime) || _workerRuntime.Equals(RpcWorkerConstants.DotNetLanguageWorkerName, StringComparison.InvariantCultureIgnoreCase))
{
// Shutdown any placeholder channels for empty function apps or dotnet function apps.
// This is needed as specilization does not kill standby placeholder channels if worker runtime is not set.
// Debouce to ensure this does not effect cold start
_shutdownStandbyWorkerChannels();
return;
}
if (functions == null || functions.Count() == 0)
{
// do not initialize function dispatcher if there are no functions
return;
}
var workerConfig = _workerConfigs.Where(c => c.Description.Language.Equals(_workerRuntime, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (workerConfig == null)
{
// Only throw if workerConfig is null AND some functions have been found.
// With .NET out-of-proc, worker config comes from functions.
throw new InvalidOperationException($"WorkerCofig for runtime: {_workerRuntime} not found");
}
_maxProcessCount = workerConfig.CountOptions.ProcessCount;
_debounceMilliSeconds = (int)workerConfig.CountOptions.ProcessStartupInterval.TotalMilliseconds;
ErrorEventsThreshold = 3 * _maxProcessCount;
if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs))
{
State = FunctionInvocationDispatcherState.Initializing;
Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);
if (webhostLanguageWorkerChannels != null)
{
foreach (string workerId in webhostLanguageWorkerChannels.Keys.ToList())
{
if (webhostLanguageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <IRpcWorkerChannel> initializedLanguageWorkerChannelTask))
{
_logger.LogDebug("Found initialized language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
try
{
IRpcWorkerChannel initializedLanguageWorkerChannel = await initializedLanguageWorkerChannelTask.Task;
initializedLanguageWorkerChannel.SetupFunctionInvocationBuffers(_functions);
initializedLanguageWorkerChannel.SendFunctionLoadRequests(_managedDependencyOptions.Value, _scriptOptions.FunctionTimeout);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Removing errored webhost language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
await _webHostLanguageWorkerChannelManager.ShutdownChannelIfExistsAsync(_workerRuntime, workerId, ex);
InitializeWebhostLanguageWorkerChannel();
}
}
}
StartWorkerProcesses(webhostLanguageWorkerChannels.Count(), InitializeWebhostLanguageWorkerChannel);
SetFunctionDispatcherStateToInitializedAndLog();
}
else
{
await InitializeJobhostLanguageWorkerChannelAsync(0);
StartWorkerProcesses(1, InitializeJobhostLanguageWorkerChannelAsync);
}
}
}
