private static FunctionDispatcher GetTestFunctionDispatcher(string maxProcessCountValue = null, bool addWebhostChannel = false, Mock <IWebHostLanguageWorkerChannelManager> mockwebHostLanguageWorkerChannelManager = null)
{
var eventManager = new ScriptEventManager();
var scriptJobHostEnvironment = new Mock <IScriptJobHostEnvironment>();
var metricsLogger = new Mock <IMetricsLogger>();
var testEnv = new TestEnvironment();
if (!string.IsNullOrEmpty(maxProcessCountValue))
{
testEnv.SetEnvironmentVariable(LanguageWorkerConstants.FunctionsWorkerProcessCountSettingName, maxProcessCountValue);
}
var loggerFactory = MockNullLoggerFactory.CreateLoggerFactory();
var options = new ScriptJobHostOptions
{
RootLogPath = Path.GetTempPath()
};
IOptions <ScriptJobHostOptions> scriptOptions = new OptionsManager <ScriptJobHostOptions>(new TestOptionsFactory <ScriptJobHostOptions>(options));
var workerConfigOptions = new LanguageWorkerOptions
{
WorkerConfigs = TestHelpers.GetTestWorkerConfigs()
};
IWebHostLanguageWorkerChannelManager testWebHostLanguageWorkerChannelManager = new TestLanguageWorkerChannelManager(eventManager, _testLogger, scriptOptions.Value.RootScriptPath);
ILanguageWorkerChannelFactory testLanguageWorkerChannelFactory = new TestLanguageWorkerChannelFactory(eventManager, _testLogger, scriptOptions.Value.RootScriptPath);
IJobHostLanguageWorkerChannelManager jobHostLanguageWorkerChannelManager = new JobHostLanguageWorkerChannelManager(loggerFactory);
if (addWebhostChannel)
{
testWebHostLanguageWorkerChannelManager.InitializeChannelAsync("java");
}
if (mockwebHostLanguageWorkerChannelManager != null)
{
testWebHostLanguageWorkerChannelManager = mockwebHostLanguageWorkerChannelManager.Object;
}
var mockFunctionDispatcherLoadBalancer = new Mock <IFunctionDispatcherLoadBalancer>();
_javaTestChannel = new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), "java", eventManager, _testLogger, false);
return(new FunctionDispatcher(scriptOptions,
metricsLogger.Object,
testEnv,
scriptJobHostEnvironment.Object,
eventManager,
loggerFactory,
testLanguageWorkerChannelFactory,
new OptionsWrapper <LanguageWorkerOptions>(workerConfigOptions),
testWebHostLanguageWorkerChannelManager,
jobHostLanguageWorkerChannelManager,
null,
mockFunctionDispatcherLoadBalancer.Object));
}
public Task<ILanguageWorkerChannel> InitializeChannelAsync(string language)
{
ILanguageWorkerChannel workerChannel = new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), language, _eventManager, _testLogger, true);
if (_workerChannels.TryGetValue(language, out List<ILanguageWorkerChannel> workerChannels))
{
workerChannels.Add(workerChannel);
}
else
{
_workerChannels.TryAdd(language, new List<ILanguageWorkerChannel>());
_workerChannels[language].Add(workerChannel);
}
workerChannel.StartWorkerProcessAsync();
return Task.FromResult(workerChannel);
}
public Task <ILanguageWorkerChannel> InitializeChannelAsync(string language)
{
ILanguageWorkerChannel workerChannel = new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), language, _eventManager, _testLogger, true);
if (_workerChannels.TryGetValue(language, out Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> > workerChannels))
{
workerChannels.Add(workerChannel.Id, new TaskCompletionSource <ILanguageWorkerChannel>());
}
else
{
_workerChannels.TryAdd(language, new Dictionary <string, TaskCompletionSource <ILanguageWorkerChannel> >());
_workerChannels[language].Add(workerChannel.Id, new TaskCompletionSource <ILanguageWorkerChannel>());
}
workerChannel.StartWorkerProcessAsync();
SetInitializedWorkerChannel(language, workerChannel);
return(Task.FromResult(workerChannel));
}
public RpcInitializationServiceTests()
{
_mockLanguageWorkerChannelManager = new Mock <IWebHostLanguageWorkerChannelManager>();
_loggerFactory = new LoggerFactory();
_logger = _loggerFactory.CreateLogger <RpcInitializationService>();
var applicationHostOptions = new ScriptApplicationHostOptions
{
IsSelfHost = true,
ScriptPath = Path.GetTempPath()
};
_optionsMonitor = TestHelpers.CreateOptionsMonitor(applicationHostOptions);
ILanguageWorkerChannel testLanguageWorkerChannel = new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), LanguageWorkerConstants.NodeLanguageWorkerName);
_mockLanguageWorkerChannelManager.Setup(m => m.InitializeChannelAsync(It.IsAny <string>()))
.Returns(Task.FromResult <ILanguageWorkerChannel>(testLanguageWorkerChannel));
}
public async void FunctionDispatcher_Restart_ErroredChannels_ExcceedsLimit()
{
int expectedProcessCount = 2;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(0, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async Task FunctionDispatcher_Error_BeyondThreshold_BucketIsAtOne()
{
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher("1");
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 1; i < 10; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerErrorWithCustomTimestamp(DateTime.UtcNow.AddHours(i));
}
}
Assert.Equal(1, functionDispatcher.LanguageWorkerErrors.Count);
}
public async Task FunctionDispatcher_Error_WithinThreshold_BucketFills()
{
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher("1");
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 0; i < 3; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(3, functionDispatcher.LanguageWorkerErrors.Count);
}
public ILanguageWorkerChannel CreateLanguageWorkerChannel(string workerId, string scriptRootPath, string language, IMetricsLogger metricsLogger, int attemptCount, bool isWebhostChannel = false, IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
{
var languageWorkerChannel = new TestLanguageWorkerChannel(workerId, language, _eventManager, _testLogger, isWebhostChannel);
if (isWebhostChannel)
{
if (_workerChannels.TryGetValue(language, out List <ILanguageWorkerChannel> workerChannels))
{
workerChannels.Add(languageWorkerChannel);
}
else
{
_workerChannels.TryAdd(language, new List <ILanguageWorkerChannel>());
_workerChannels[language].Add(languageWorkerChannel);
}
}
return(languageWorkerChannel);
}
public async void FunctionDispatcher_Restart_ErroredChannels_Succeeds()
{
int expectedProcessCount = 2;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
int finalChannelCount = 0;
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
TestLanguageWorkerChannel testWorkerChannel = (TestLanguageWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
if (functionDispatcher.LanguageWorkerErrors.Count < (expectedProcessCount * 3) - 1)
{
testWorkerChannel.RaiseWorkerError();
}
finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
}
Assert.Equal(expectedProcessCount, finalChannelCount);
}
public async Task FunctionDispatcher_Restart_ErroredChannels_OnWorkerRestart_NotAffectedByLimit()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerRestart();
}
var finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Assert.Equal(expectedProcessCount, finalChannelCount);
}
}