public async Task FunctionDispatcher_DoNot_Restart_ErroredChannels_If_WorkerRuntime_DoesNotMatch()
{
int expectedProcessCount = 1;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
_javaTestChannel.RaiseWorkerError();
var testLogs = _testLogger.GetLogMessages();
Assert.False(testLogs.Any(m => m.FormattedMessage.Contains("Restarting worker channel for runtime:java")));
Assert.Equal(expectedProcessCount, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async void FunctionDispatcher_Restart_ErroredChannels_ExcceedsLimit()
{
int expectedProcessCount = 2;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(0, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async Task FunctionDispatcher_Error_WithinThreshold_BucketFills()
{
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher("1");
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 0; i < 3; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(3, functionDispatcher.LanguageWorkerErrors.Count);
}
public async void FunctionDispatcher_Restart_ErroredChannels_Succeeds()
{
int expectedProcessCount = 2;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
int finalChannelCount = 0;
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
TestLanguageWorkerChannel testWorkerChannel = (TestLanguageWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
if (functionDispatcher.LanguageWorkerErrors.Count < (expectedProcessCount * 3) - 1)
{
testWorkerChannel.RaiseWorkerError();
}
finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
}
Assert.Equal(expectedProcessCount, finalChannelCount);
}