public void ShutdownStandbyChannels_OnlyProxies()
{
_testEnvironment = new TestEnvironment();
_languageWorkerChannelManager = new LanguageWorkerChannelManager(_eventManager, _testEnvironment, _rpcServer, _loggerFactory, new OptionsWrapper <LanguageWorkerOptions>(_languageWorkerOptions), _optionsMonitor);
string javaWorkerId = Guid.NewGuid().ToString();
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(javaWorkerId, LanguageWorkerConstants.JavaLanguageWorkerName);
FunctionMetadata proxy1 = new FunctionMetadata()
{
Name = "funcproxy1",
IsProxy = true
};
FunctionMetadata proxy2 = new FunctionMetadata()
{
Name = "funcproxy2",
IsProxy = true
};
IEnumerable <FunctionMetadata> functionsList = new Collection <FunctionMetadata>()
{
proxy2, proxy1
};
_languageWorkerChannelManager.ShutdownStandbyChannels(functionsList);
var initializedChannel = _languageWorkerChannelManager.GetChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public async Task SpecializeAsync()
{
_logger.LogInformation("Starting language worker channel specialization");
_workerRuntime = _environment.GetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName);
ILanguageWorkerChannel languageWorkerChannel = await GetChannelAsync(_workerRuntime);
if (_workerRuntime != null && languageWorkerChannel != null)
{
if (UsePlaceholderChannel(_workerRuntime))
{
_logger.LogDebug("Loading environment variables for runtime: {runtime}", _workerRuntime);
await languageWorkerChannel.SendFunctionEnvironmentReloadRequest();
}
else
{
_logger.LogDebug("Shutting down placeholder worker. Worker is not compatible for runtime: {runtime}", _workerRuntime);
// If we need to allow file edits, we should shutdown the webhost channel on specialization.
await ShutdownChannelIfExistsAsync(_workerRuntime, languageWorkerChannel.Id);
}
}
_shutdownStandbyWorkerChannels();
_logger.LogDebug("Completed language worker channel specialization");
}
public Task <ILanguageWorkerChannel> InitializeChannelAsync(string language)
{
ILanguageWorkerChannel workerChannel = CreateLanguageWorkerChannel(Guid.NewGuid().ToString(), _scriptRootPath, language, null, 0, true);
workerChannel.StartWorkerProcessAsync();
return(Task.FromResult(workerChannel));
}
public async Task <ILanguageWorkerChannel> InitializeChannelAsync(string language)
{
var metricsLogger = new Mock <IMetricsLogger>();
ILanguageWorkerChannel workerChannel = _testLanguageWorkerChannelFactory.CreateLanguageWorkerChannel(_scriptRootPath, language, metricsLogger.Object, 0);
if (_workerChannels.TryGetValue(language, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > workerChannels))
{
workerChannels.Add(workerChannel.Id, new TaskCompletionSource <ILanguageWorkerChannel>());
}
else
{
_workerChannels.TryAdd(language, new Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> >());
_workerChannels[language].Add(workerChannel.Id, new TaskCompletionSource <ILanguageWorkerChannel>());
}
await workerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask =>
{
if (processStartTask.Status == TaskStatus.RanToCompletion)
{
SetInitializedWorkerChannel(language, workerChannel);
}
else if (processStartTask.Status == TaskStatus.Faulted)
{
SetExceptionOnInitializedWorkerChannel(language, workerChannel, processStartTask.Exception);
}
});
return(workerChannel);
}
internal async void InitializeWebhostLanguageWorkerChannel()
{
_logger.LogDebug("Creating new webhost language worker channel for runtime:{workerRuntime}.", _workerRuntime);
ILanguageWorkerChannel workerChannel = await _languageWorkerChannelManager.InitializeChannelAsync(_workerRuntime);
workerChannel.RegisterFunctions(_workerState.Functions);
}
public void DisposeAndRemoveChannel(ILanguageWorkerChannel channel)
{
if (_channels.TryRemove(channel.Id, out ILanguageWorkerChannel removedChannel))
{
(removedChannel as IDisposable)?.Dispose();
}
}
public async Task <bool> ShutdownChannelIfExistsAsync(string language, string workerId)
{
if (string.IsNullOrEmpty(language))
{
throw new ArgumentNullException(nameof(language));
}
if (_workerChannels.TryGetValue(language, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > languageWorkerChannels))
{
if (languageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <ILanguageWorkerChannel> value))
{
try
{
ILanguageWorkerChannel channel = await value?.Task;
if (channel != null)
{
(channel as IDisposable)?.Dispose();
languageWorkerChannels.Remove(workerId);
return(true);
}
}
catch (Exception)
{
languageWorkerChannels.Remove(workerId);
return(true);
}
}
}
return(false);
}
internal async Task <ILanguageWorkerChannel> InitializeLanguageWorkerChannel(string runtime, string scriptRootPath)
{
ILanguageWorkerChannel languageWorkerChannel = null;
string workerId = Guid.NewGuid().ToString();
_logger.LogDebug("Creating language worker channel for runtime:{runtime}", runtime);
try
{
languageWorkerChannel = _languageWorkerChannelFactory.CreateLanguageWorkerChannel(scriptRootPath, runtime, null, 0);
AddOrUpdateWorkerChannels(runtime, languageWorkerChannel);
await languageWorkerChannel.StartWorkerProcessAsync().ContinueWith(processStartTask =>
{
if (processStartTask.Status == TaskStatus.RanToCompletion)
{
_logger.LogDebug("Adding jobhost language worker channel for runtime: {language}. workerId:{id}", _workerRuntime, languageWorkerChannel.Id);
SetInitializedWorkerChannel(runtime, languageWorkerChannel);
}
else if (processStartTask.Status == TaskStatus.Faulted)
{
_logger.LogError("Failed to start language worker process for runtime: {language}. workerId:{id}", _workerRuntime, languageWorkerChannel.Id);
SetExceptionOnInitializedWorkerChannel(runtime, languageWorkerChannel, processStartTask.Exception);
}
});
}
catch (Exception ex)
{
throw new HostInitializationException($"Failed to start Language Worker Channel for language :{runtime}", ex);
}
return(languageWorkerChannel);
}
public Task ShutdownChannelsAsync()
{
foreach (string runtime in _workerChannels.Keys)
{
_logger.LogInformation("Shutting down language worker channels for runtime:{runtime}", runtime);
if (_workerChannels.TryRemove(runtime, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > standbyChannels))
{
foreach (string workerId in standbyChannels.Keys)
{
standbyChannels[workerId]?.Task.ContinueWith(channelTask =>
{
if (channelTask.Status == TaskStatus.Faulted)
{
_logger.LogDebug(channelTask.Exception, "Removing errored worker channel");
}
else
{
ILanguageWorkerChannel workerChannel = channelTask.Result;
if (workerChannel != null)
{
(channelTask.Result as IDisposable)?.Dispose();
}
}
});
}
}
}
return(Task.CompletedTask);
}
internal void ScheduleShutdownStandbyChannels()
{
_workerRuntime = _workerRuntime ?? _environment.GetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName);
if (!string.IsNullOrEmpty(_workerRuntime))
{
var standbyWorkerChannels = _workerChannels.Where(ch => !ch.Key.Equals(_workerRuntime, StringComparison.InvariantCultureIgnoreCase));
foreach (var runtime in standbyWorkerChannels)
{
_logger.LogInformation("Disposing standby channel for runtime:{language}", runtime.Key);
if (_workerChannels.TryRemove(runtime.Key, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > standbyChannels))
{
foreach (string workerId in standbyChannels.Keys)
{
standbyChannels[workerId]?.Task.ContinueWith(channelTask =>
{
if (channelTask.Status == TaskStatus.Faulted)
{
_logger.LogDebug(channelTask.Exception, "Removing errored worker channel");
}
else
{
ILanguageWorkerChannel workerChannel = channelTask.Result;
if (workerChannel != null)
{
(channelTask.Result as IDisposable)?.Dispose();
}
}
});
}
}
}
}
}
public Task <bool> ShutdownChannelIfExistsAsync(string language, string workerId)
{
if (string.IsNullOrEmpty(language))
{
throw new ArgumentNullException(nameof(language));
}
if (_workerChannels.TryRemove(language, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > languageWorkerChannels))
{
if (languageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <ILanguageWorkerChannel> value))
{
value?.Task.ContinueWith(channelTask =>
{
if (channelTask.Status == TaskStatus.Faulted)
{
_logger.LogDebug(channelTask.Exception, "Removing errored worker channel");
}
else
{
ILanguageWorkerChannel workerChannel = channelTask.Result;
if (workerChannel != null)
{
(channelTask.Result as IDisposable)?.Dispose();
}
}
});
return(Task.FromResult(true));
}
}
return(Task.FromResult(false));
}
internal void DisposeAndRemoveChannel(ILanguageWorkerChannel channel)
{
if (_channels.TryRemove(channel.Id, out ILanguageWorkerChannel removedChannel))
{
channel?.Dispose();
}
}
public void ShutdownStandyChannels_WorkerRuntime_Node_Set()
{
_testEnvironment = new TestEnvironment();
_testEnvironment.SetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName, LanguageWorkerConstants.NodeLanguageWorkerName);
_languageWorkerChannelManager = new LanguageWorkerChannelManager(_eventManager, _testEnvironment, _rpcServer, _loggerFactory, new OptionsWrapper <LanguageWorkerOptions>(_languageWorkerOptions), _optionsMonitor);
string javaWorkerId = Guid.NewGuid().ToString();
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(javaWorkerId, LanguageWorkerConstants.JavaLanguageWorkerName);
FunctionMetadata funcJs1 = new FunctionMetadata()
{
Name = "funcJs1",
Language = "node"
};
FunctionMetadata funcCS1 = new FunctionMetadata()
{
Name = "funcCS1",
Language = "csharp"
};
IEnumerable <FunctionMetadata> functionsList = new Collection <FunctionMetadata>()
{
funcJs1, funcCS1
};
_languageWorkerChannelManager.ShutdownStandbyChannels(functionsList);
var initializedChannel = _languageWorkerChannelManager.GetChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public void DisposeAndRemoveChannel(ILanguageWorkerChannel channel)
{
if (_channels.TryRemove(channel.Id, out ILanguageWorkerChannel removedChannel))
{
_logger.LogDebug("Disposing language worker channel with id:{workerId}", removedChannel.Id);
(removedChannel as IDisposable)?.Dispose();
}
}
internal async void InitializeWebhostLanguageWorkerChannel()
{
_logger.LogDebug("Creating new webhost language worker channel for runtime:{workerRuntime}.", _workerRuntime);
ILanguageWorkerChannel workerChannel = await _webHostLanguageWorkerChannelManager.InitializeChannelAsync(_workerRuntime);
workerChannel.SetupFunctionInvocationBuffers(_functions);
workerChannel.SendFunctionLoadRequests();
}
internal RpcChannelReadyEvent(string id, string language, ILanguageWorkerChannel languageWorkerChannel,
string version, IDictionary <string, string> capabilities)
: base(id)
{
Language = language;
LanguageWorkerChannel = languageWorkerChannel;
Version = version;
Capabilities = capabilities;
}
internal void SetInitializedWorkerChannel(string initializedRuntime, ILanguageWorkerChannel initializedLanguageWorkerChannel)
{
if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > channel))
{
if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource <ILanguageWorkerChannel> value))
{
value.SetResult(initializedLanguageWorkerChannel);
}
}
}
public ILanguageWorkerChannel GetChannel(string language)
{
ILanguageWorkerChannel initializedChannel = null;
if (!string.IsNullOrEmpty(language) && _workerChannels.TryGetValue(language, out initializedChannel))
{
return(initializedChannel);
}
return(null);
}
public void ShutdownStandbyChannels_WorkerRuntime_Not_Set()
{
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
_languageWorkerChannelManager.ShutdownChannels();
var initializedChannel = _languageWorkerChannelManager.GetChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public async Task ShutdownStandbyChannels_WorkerRuntime_Not_Set()
{
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
await _languageWorkerChannelManager.ShutdownChannelsAsync();
ILanguageWorkerChannel initializedChannel = await _languageWorkerChannelManager.GetChannelAsync(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
internal void SetInitializedWorkerChannel(string initializedRuntime, ILanguageWorkerChannel initializedLanguageWorkerChannel)
{
_logger.LogDebug("Adding webhost language worker channel for runtime: {language}. workerId:{id}", initializedRuntime, initializedLanguageWorkerChannel.Id);
if (_workerChannels.TryGetValue(initializedRuntime, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > channel))
{
if (channel.TryGetValue(initializedLanguageWorkerChannel.Id, out TaskCompletionSource <ILanguageWorkerChannel> value))
{
value.SetResult(initializedLanguageWorkerChannel);
}
}
}
public async Task InitializeAsync_WorkerRuntime_Node_DoNotInitialize_JavaWorker()
{
var channelManager = _languageWorkerChannelManager as WebHostLanguageWorkerChannelManager;
ILanguageWorkerChannel javaChannel = await channelManager.GetChannelAsync(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(javaChannel);
ILanguageWorkerChannel nodeChannel = await channelManager.GetChannelAsync(LanguageWorkerConstants.NodeLanguageWorkerName);
Assert.Null(nodeChannel);
}
public void CreateChannels_Succeeds()
{
string language = LanguageWorkerConstants.JavaLanguageWorkerName;
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(language);
var initializedChannel = _languageWorkerChannelManager.GetChannel(language);
ILanguageWorkerChannel javaWorkerChannel2 = CreateTestChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.NotNull(initializedChannel);
Assert.Equal(javaWorkerChannel.Id, initializedChannel.Id);
Assert.Equal(_languageWorkerChannelManager.GetChannels(LanguageWorkerConstants.JavaLanguageWorkerName).Count(), 2);
}
public void CreateChannel_Succeeds()
{
_languageWorkerChannelManager = new LanguageWorkerChannelManager(_eventManager, _testEnvironment, _rpcServer, _loggerFactory, new OptionsWrapper <LanguageWorkerOptions>(_languageWorkerOptions), _optionsMonitor);
string workerId = Guid.NewGuid().ToString();
string language = LanguageWorkerConstants.JavaLanguageWorkerName;
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(workerId, language);
var initializedChannel = _languageWorkerChannelManager.GetChannel(language);
Assert.NotNull(initializedChannel);
Assert.Equal(workerId, initializedChannel.Id);
}
public async Task SpecializeAsync()
{
_logger.LogInformation("Starting language worker channel specialization");
_workerRuntime = _environment.GetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName);
ILanguageWorkerChannel languageWorkerChannel = GetChannel(_workerRuntime);
if (_workerRuntime != null && languageWorkerChannel != null)
{
_logger.LogInformation("Loading environment variables for runtime: {runtime}", _workerRuntime);
await languageWorkerChannel.SendFunctionEnvironmentReloadRequest();
}
_shutdownStandbyWorkerChannels();
}
public async Task ShutdownStandyChannels_WorkerRuntime_Node_Set()
{
_testEnvironment.SetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName, LanguageWorkerConstants.NodeLanguageWorkerName);
_languageWorkerChannelManager = new WebHostLanguageWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _languageWorkerChannelFactory, _optionsMonitor);
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
_languageWorkerChannelManager.ScheduleShutdownStandbyChannels();
var initializedChannel = await _languageWorkerChannelManager.GetChannelAsync(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public async Task InitializeAsync(IEnumerable <FunctionMetadata> functions)
{
if (_environment.IsPlaceholderModeEnabled())
{
return;
}
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions);
_functions = functions;
if (string.IsNullOrEmpty(_workerRuntime) || _workerRuntime.Equals(LanguageWorkerConstants.DotNetLanguageWorkerName, StringComparison.InvariantCultureIgnoreCase))
{
// Shutdown any placeholder channels for empty function apps or dotnet function apps.
// This is needed as specilization does not kill standby placeholder channels if worker runtime is not set.
// Debouce to ensure this does not effect cold start
_shutdownStandbyWorkerChannels();
return;
}
if (functions == null || functions.Count() == 0)
{
// do not initialize function dispachter if there are no functions
return;
}
if (Utility.IsSupportedRuntime(_workerRuntime, _workerConfigs))
{
State = FunctionDispatcherState.Initializing;
Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > webhostLanguageWorkerChannels = _webHostLanguageWorkerChannelManager.GetChannels(_workerRuntime);
if (webhostLanguageWorkerChannels != null)
{
foreach (string workerId in webhostLanguageWorkerChannels.Keys)
{
if (webhostLanguageWorkerChannels.TryGetValue(workerId, out TaskCompletionSource <ILanguageWorkerChannel> initializedLanguageWorkerChannelTask))
{
_logger.LogDebug("Found initialized language worker channel for runtime: {workerRuntime} workerId:{workerId}", _workerRuntime, workerId);
ILanguageWorkerChannel initializedLanguageWorkerChannel = await initializedLanguageWorkerChannelTask.Task;
initializedLanguageWorkerChannel.SetupFunctionInvocationBuffers(_functions);
initializedLanguageWorkerChannel.SendFunctionLoadRequests();
}
}
StartWorkerProcesses(webhostLanguageWorkerChannels.Count(), InitializeWebhostLanguageWorkerChannel);
State = FunctionDispatcherState.Initialized;
}
else
{
await InitializeJobhostLanguageWorkerChannelAsync(0);
StartWorkerProcesses(1, InitializeJobhostLanguageWorkerChannelAsync);
}
}
}
public void ShutdownStandbyChannels_WorkerRuntime_Not_Set()
{
_testEnvironment = new TestEnvironment();
_languageWorkerChannelManager = new LanguageWorkerChannelManager(_eventManager, _testEnvironment, _rpcServer, _loggerFactory, new OptionsWrapper <LanguageWorkerOptions>(_languageWorkerOptions), _optionsMonitor, null);
string javaWorkerId = Guid.NewGuid().ToString();
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(javaWorkerId, LanguageWorkerConstants.JavaLanguageWorkerName);
_languageWorkerChannelManager.ShutdownChannels();
var initializedChannel = _languageWorkerChannelManager.GetChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public void ShutdownChannelsIfExist_Succeeds()
{
ILanguageWorkerChannel javaWorkerChannel1 = CreateTestChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
ILanguageWorkerChannel javaWorkerChannel2 = CreateTestChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
_languageWorkerChannelManager.ShutdownChannelIfExists(LanguageWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel1.Id);
_languageWorkerChannelManager.ShutdownChannelIfExists(LanguageWorkerConstants.JavaLanguageWorkerName, javaWorkerChannel2.Id);
Assert.Empty(_languageWorkerChannelManager.GetChannels(LanguageWorkerConstants.JavaLanguageWorkerName));
var initializedChannel = _languageWorkerChannelManager.GetChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
public void ShutdownStandByChannels_WorkerRuntinmeDotNet_Succeeds()
{
_testEnvironment.SetEnvironmentVariable(LanguageWorkerConstants.FunctionWorkerRuntimeSettingName, LanguageWorkerConstants.DotNetLanguageWorkerName);
_languageWorkerChannelManager = new WebHostLanguageWorkerChannelManager(_eventManager, _testEnvironment, _loggerFactory, _languageWorkerChannelFactory, _optionsMonitor);
ILanguageWorkerChannel javaWorkerChannel = CreateTestChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
ILanguageWorkerChannel nodeWorkerChannel = CreateTestChannel(LanguageWorkerConstants.NodeLanguageWorkerName);
_languageWorkerChannelManager.ScheduleShutdownStandbyChannels();
var initializedChannel = _languageWorkerChannelManager.GetChannel(LanguageWorkerConstants.NodeLanguageWorkerName);
Assert.Null(initializedChannel);
initializedChannel = _languageWorkerChannelManager.GetChannel(LanguageWorkerConstants.JavaLanguageWorkerName);
Assert.Null(initializedChannel);
}
