public async Task FunctionDispatcherState_Default_DotNetFunctions()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher();
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "dotnet"
};
var functions = new List <FunctionMetadata>()
{
func1
};
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
}
public async Task FunctionDispatcherState_Default_NoFunctions()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(runtime: RpcWorkerConstants.NodeLanguageWorkerName);
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
}
public async Task FunctionDispatcherState_Default_NoFunctions()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher();
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
}
public async Task SuccessiveRestarts_WorkerCountsStayTheSame()
{
int expectedProcessCount = 3;
List <Task> restartTasks = new List <Task>();
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Guid[] invocationIds = new Guid[] { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() };
List <TestRpcWorkerChannel> workerChannels = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Cast <TestRpcWorkerChannel>().ToList();
for (int i = 0; i < invocationIds.Length; ++i)
{
workerChannels[i % 3].SendInvocationRequest(new ScriptInvocationContext
{
ExecutionContext = new ExecutionContext
{
InvocationId = invocationIds[i]
}
});
}
foreach (var invocationId in invocationIds)
{
restartTasks.Add(functionDispatcher.RestartWorkerWithInvocationIdAsync(invocationId.ToString()));
}
await Task.WhenAll(restartTasks);
Assert.Equal(expectedProcessCount, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count()); // Ensure count always stays at the initial count
}
public async Task Restart_ParticularWorkerChannel_Succeeds_OnlyThatIsDisposed()
{
int expectedProcessCount = 3;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Guid invocationId = Guid.NewGuid();
List <TestRpcWorkerChannel> workerChannels = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Cast <TestRpcWorkerChannel>().ToList();
workerChannels[0].SendInvocationRequest(new ScriptInvocationContext
{
ExecutionContext = new ExecutionContext
{
InvocationId = invocationId
}
});
await functionDispatcher.RestartWorkerWithInvocationIdAsync(invocationId.ToString());
Assert.True(workerChannels[0].IsDisposed);
for (int i = 1; i < workerChannels.Count; ++i)
{
Assert.False(workerChannels[i].IsDisposed); // Ensure no other channel is disposed
}
Assert.Equal(expectedProcessCount, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count()); // Ensure count goes back to initial count
}
public async Task FunctionDispatcher_ShouldRestartChannel_Returns_True(string workerRuntime, string channelLanguage, bool isWebHostChannel, bool isJobHostChannel, bool expectedResult)
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher();
await functionDispatcher.InitializeAsync(GetTestFunctionsList(workerRuntime));
Assert.Equal(expectedResult, functionDispatcher.ShouldRestartWorkerChannel(channelLanguage, isWebHostChannel, isJobHostChannel));
}
public async Task WorkerRuntime_Setting_ChannelInitializationState_Succeeds(string workerRuntime)
{
_testLoggerProvider.ClearAllLogMessages();
int expectedProcessCount = 1;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount, false, runtime: workerRuntime, workerIndexing: true);
// create channels and ensure that they aren't ready for invocation requests yet
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
if (!string.IsNullOrEmpty(workerRuntime))
{
int createdChannelsCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount, false);
Assert.Equal(expectedProcessCount, createdChannelsCount);
IEnumerable <IRpcWorkerChannel> channels = await functionDispatcher.GetInitializedWorkerChannelsAsync();
Assert.Equal(0, channels.Count());
// set up invocation buffers, send load requests, and ensure that the channels are now set up for invocation requests
var functions = GetTestFunctionsList(RpcWorkerConstants.JavaLanguageWorkerName);
await functionDispatcher.FinishInitialization(functions);
int initializedChannelsCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount, true);
Assert.Equal(expectedProcessCount, initializedChannelsCount);
}
else
{
foreach (var currChannel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
Assert.True(((TestRpcWorkerChannel)currChannel).ExecutionContexts.Count == 0);
}
}
}
public async Task FunctionDispatcher_ErroredWebHostChannel()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(throwOnProcessStartUp: true, addWebhostChannel: true);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.JavaLanguageWorkerName));
var testLogs = _testLogger.GetLogMessages();
Assert.False(testLogs.Any(m => m.FormattedMessage.Contains("Removing errored webhost language worker channel for runtime")));
}
public async Task Starting_MultipleJobhostChannels_Succeeds()
{
int expectedProcessCount = 3;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
var finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Assert.Equal(expectedProcessCount, finalChannelCount);
}
public async Task FunctionDispatcherState_Initialized_RemainsInitializing(bool setException = false)
{
var mockRpcWorkerChannel = new Mock <IRpcWorkerChannel>();
var tcs = new TaskCompletionSource <IRpcWorkerChannel>();
Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > webhostLanguageWorkerChannels = new Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> >();
Mock <IWebHostRpcWorkerChannelManager> mockWebHostChannelManager = new Mock <IWebHostRpcWorkerChannelManager>();
mockRpcWorkerChannel.Setup(a => a.StartWorkerProcessAsync(CancellationToken.None)).Returns(Task.FromResult(true));
mockRpcWorkerChannel.Setup(a => a.SetupFunctionInvocationBuffers(It.IsAny <IEnumerable <FunctionMetadata> >()));
mockRpcWorkerChannel.Setup(a => a.SendFunctionLoadRequests(It.IsAny <ManagedDependencyOptions>(), It.IsAny <TimeSpan?>()));
webhostLanguageWorkerChannels.Add("java", tcs);
mockWebHostChannelManager.Setup(a => a.GetChannels(It.IsAny <string>())).Returns(webhostLanguageWorkerChannels);
mockWebHostChannelManager.Setup(a => a.ShutdownChannelIfExistsAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <Exception>())).Returns(Task.FromResult(true));
if (!setException)
{
tcs.SetResult(mockRpcWorkerChannel.Object);
}
else
{
tcs.SetException(new Exception());
}
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(addWebhostChannel: true, mockwebHostLanguageWorkerChannelManager: mockWebHostChannelManager, runtime: RpcWorkerConstants.JavaLanguageWorkerName);
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "node"
};
var functions = new List <FunctionMetadata>()
{
func1
};
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionInvocationDispatcherState.Initializing, functionDispatcher.State);
try
{
await WaitForFunctionDispactherStateInitialized(functionDispatcher);
}
catch (Exception)
{
// We don't care if this times out
}
if (!setException)
{
Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
}
else
{
Assert.Equal(FunctionInvocationDispatcherState.Initializing, functionDispatcher.State);
}
}
public async Task Starting_MultipleJobhostChannels_Succeeds()
{
_testLoggerProvider.ClearAllLogMessages();
int expectedProcessCount = 3;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount, startupIntervals: TimeSpan.FromSeconds(1), runtime: RpcWorkerConstants.NodeLanguageWorkerName);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
var finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Assert.Equal(expectedProcessCount, finalChannelCount);
VerifyStartIntervals(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2));
}
public async Task FunctionDispatcher_DoNot_Restart_ErroredChannels_If_WorkerRuntime_DoesNotMatch()
{
int expectedProcessCount = 1;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
_javaTestChannel.RaiseWorkerError();
var testLogs = _testLogger.GetLogMessages();
Assert.False(testLogs.Any(m => m.FormattedMessage.Contains("Restarting worker channel for runtime:java")));
Assert.Equal(expectedProcessCount, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async Task Starting_MultipleWebhostChannels_Succeeds()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount, true);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.JavaLanguageWorkerName));
var finalWebhostChannelCount = await WaitForWebhostWorkerChannelsToStartup(functionDispatcher.WebHostLanguageWorkerChannelManager, expectedProcessCount, "java");
Assert.Equal(expectedProcessCount, finalWebhostChannelCount);
var finalJobhostChannelCount = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count();
Assert.Equal(0, finalJobhostChannelCount);
}
public async Task ShutdownChannels_NoFunctions()
{
var mockLanguageWorkerChannelManager = new Mock <IWebHostRpcWorkerChannelManager>();
mockLanguageWorkerChannelManager.Setup(m => m.ShutdownChannelsAsync()).Returns(Task.CompletedTask);
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(mockwebHostLanguageWorkerChannelManager: mockLanguageWorkerChannelManager);
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
// Wait longer than debouce action.
await Task.Delay(6000);
mockLanguageWorkerChannelManager.Verify(m => m.ShutdownChannelsAsync(), Times.Once);
}
public async Task WorkerIndexing_Starting_MultipleWebhostChannels_Succeeds()
{
_testLoggerProvider.ClearAllLogMessages();
int expectedProcessCount = 3;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount, true, runtime: RpcWorkerConstants.JavaLanguageWorkerName, workerIndexing: true);
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
var finalWebhostChannelCount = await WaitForWebhostWorkerChannelsToStartup(functionDispatcher.WebHostLanguageWorkerChannelManager, expectedProcessCount, RpcWorkerConstants.JavaLanguageWorkerName);
Assert.Equal(expectedProcessCount, finalWebhostChannelCount);
var finalJobhostChannelCount = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count();
Assert.Equal(0, finalJobhostChannelCount);
}
public async Task Starting_MultipleJobhostChannels_Failed()
{
_testLoggerProvider.ClearAllLogMessages();
int expectedProcessCount = 3;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount, throwOnProcessStartUp: true);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
var finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount, false);
Assert.Equal(expectedProcessCount, finalChannelCount);
var logMessages = _testLoggerProvider.GetAllLogMessages().ToList();
Assert.Equal(logMessages.Where(x => x.FormattedMessage
.Contains("Failed to start a new language worker")).Count(), 3);
}
public async Task FunctionDispatcherState_Transitions_From_Starting_To_Initialized()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher();
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "node"
};
var functions = new List <FunctionMetadata>()
{
func1
};
await functionDispatcher.InitializeAsync(functions);
Assert.True(functionDispatcher.State == FunctionDispatcherState.Initializing || functionDispatcher.State == FunctionDispatcherState.Initialized);
await WaitForFunctionDispactherStateInitialized(functionDispatcher);
}
public async Task Starting_MultipleWebhostChannels_Succeeds()
{
_testLoggerProvider.ClearAllLogMessages();
int expectedProcessCount = 3;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount, true, runtime: RpcWorkerConstants.JavaLanguageWorkerName);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.JavaLanguageWorkerName));
var finalWebhostChannelCount = await WaitForWebhostWorkerChannelsToStartup(functionDispatcher.WebHostLanguageWorkerChannelManager, expectedProcessCount, "java");
Assert.Equal(expectedProcessCount, finalWebhostChannelCount);
var finalJobhostChannelCount = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count();
Assert.Equal(0, finalJobhostChannelCount);
// ignore first start as we added a WebhostChannel on GetTestFunctionDispatcher call
VerifyStartIntervals(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(15), true);
}
public async Task FunctionDispatcher_Error_BeyondThreshold_BucketIsAtOne()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher();
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 1; i < 10; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel;
testWorkerChannel.RaiseWorkerErrorWithCustomTimestamp(DateTime.UtcNow.AddHours(i));
}
}
Assert.Equal(1, functionDispatcher.LanguageWorkerErrors.Count);
}
public async Task FunctionDispatcher_Restart_ErroredChannels_ExceedsLimit()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(0, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async Task FunctionDispatcher_Error_WithinThreshold_BucketFills()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(1);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 0; i < 3; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestRpcWorkerChannel testWorkerChannel = channel as TestRpcWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(3, functionDispatcher.LanguageWorkerErrors.Count);
}
public async Task Restart_AllChannels_Succeeds()
{
int expectedProcessCount = 3;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Guid invocationId = Guid.NewGuid();
List <TestRpcWorkerChannel> workerChannels = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Cast <TestRpcWorkerChannel>().ToList();
await functionDispatcher.RestartAllWorkersAsync();
foreach (TestRpcWorkerChannel channel in workerChannels)
{
Assert.True(channel.IsDisposed);
}
Assert.Equal(expectedProcessCount, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count()); // Ensure count goes back to initial count
}
public async Task FunctionDispatcherState_Transitions_From_Starting_To_Initialized()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(runtime: RpcWorkerConstants.NodeLanguageWorkerName);
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "node"
};
var functions = new List <FunctionMetadata>()
{
func1
};
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionInvocationDispatcherState.Initializing, functionDispatcher.State);
await WaitForFunctionDispactherStateInitialized(functionDispatcher);
Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
}
public async Task FunctionDispatcher_Restart_ErroredChannels_And_DoesNot_Change_State()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
// Add worker
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
// Restart one channel
testWorkerChannel.RaiseWorkerRestart();
Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
}
public async Task ShutdownTests()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
foreach (var currChannel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
var initializedChannel = (TestRpcWorkerChannel)currChannel;
initializedChannel.ExecutionContexts.Add(Task.Factory.StartNew(() => { }));
}
await functionDispatcher.ShutdownAsync();
foreach (var currChannel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
Assert.True(((TestRpcWorkerChannel)currChannel).ExecutionContexts.Count == 0);
}
}
public async Task FunctionDispatcher_Restart_ErroredChannels_Succeeds()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
int finalChannelCount = 0;
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
if (functionDispatcher.LanguageWorkerErrors.Count < (expectedProcessCount * 3) - 1)
{
testWorkerChannel.RaiseWorkerError();
}
finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
}
Assert.Equal(expectedProcessCount, finalChannelCount);
}
public async Task ShutdownTests_WithInfinitelyRunningTasks_Timesout()
{
int expectedProcessCount = 2;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
foreach (var currChannel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
var initializedChannel = (TestRpcWorkerChannel)currChannel;
initializedChannel.ExecutionContexts.Add(new Task <bool>(() => true)); // A task that never starts and therefore never runs to completion
}
await functionDispatcher.ShutdownAsync();
foreach (var currChannel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
Assert.True(((TestRpcWorkerChannel)currChannel).ExecutionContexts.Count > 0);
}
}
public async Task ShutdownChannels_DotNetFunctions()
{
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "dotnet"
};
var functions = new List <FunctionMetadata>()
{
func1
};
var mockLanguageWorkerChannelManager = new Mock <IWebHostRpcWorkerChannelManager>();
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(mockwebHostLanguageWorkerChannelManager: mockLanguageWorkerChannelManager);
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(functions);
// Wait longer than debouce action.
await Task.Delay(6000);
mockLanguageWorkerChannelManager.Verify(m => m.ShutdownChannelsAsync(), Times.Once);
}
public async Task FunctionDispatcherState_Transitions_From_Default_To_Initialized_To_Disposing()
{
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher();
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "node"
};
var functions = new List <FunctionMetadata>()
{
func1
};
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionInvocationDispatcherState.Initializing, functionDispatcher.State);
await WaitForFunctionDispactherStateInitialized(functionDispatcher);
Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
functionDispatcher.Dispose();
Assert.True(functionDispatcher == null || functionDispatcher.State == FunctionInvocationDispatcherState.Disposing || functionDispatcher.State == FunctionInvocationDispatcherState.Disposed);
}
public async Task FunctionDispatcher_Restart_ErroredChannels_And_Changes_State()
{
int expectedProcessCount = 1;
RpcFunctionInvocationDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString());
Assert.Equal(FunctionInvocationDispatcherState.Default, functionDispatcher.State);
// Add worker
await functionDispatcher.InitializeAsync(GetTestFunctionsList(RpcWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
TestRpcWorkerChannel testWorkerChannel = (TestRpcWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
// Restart this channel
testWorkerChannel.RaiseWorkerRestart();
await TestHelpers.Await(() =>
{
return(functionDispatcher.State == FunctionInvocationDispatcherState.WorkerProcessRestarting);
}, 3000);
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Assert.Equal(FunctionInvocationDispatcherState.Initialized, functionDispatcher.State);
}
