public async Task FunctionDispatcher_DoNot_Restart_ErroredChannels_If_WorkerRuntime_DoesNotMatch()
{
int expectedProcessCount = 1;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
_javaTestChannel.RaiseWorkerError();
var testLogs = _testLogger.GetLogMessages();
Assert.False(testLogs.Any(m => m.FormattedMessage.Contains("Restarting worker channel for runtime:java")));
Assert.Equal(expectedProcessCount, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async Task FunctionDispatcher_Restart_ErroredChannels_ExceedsLimit()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(0, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async Task FunctionDispatcher_Error_WithinThreshold_BucketFills()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(1);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 0; i < 3; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(3, functionDispatcher.LanguageWorkerErrors.Count);
}
public async Task FunctionDispatcher_Restart_ErroredChannels_Succeeds()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
int finalChannelCount = 0;
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
if (functionDispatcher.LanguageWorkerErrors.Count < (expectedProcessCount * 3) - 1)
{
testWorkerChannel.RaiseWorkerError();
}
finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
}
Assert.Equal(expectedProcessCount, finalChannelCount);
}
