public async void FunctionDispatcherState_Default_DotNetFunctions()
{
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher();
Assert.Equal(FunctionDispatcherState.Default, functionDispatcher.State);
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "dotnet"
};
var functions = new List <FunctionMetadata>()
{
func1
};
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(functions);
Assert.Equal(FunctionDispatcherState.Default, functionDispatcher.State);
}
public void MaxProcessCount_ProcessCount_Set_ExceedsMax_Returns_ExpectedCount()
{
int expectedProcessCount = 30;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
Assert.Equal(functionDispatcher.MaxProcessCount, 10);
}
public async Task FunctionDispatcher_ShouldRestartChannel_Returns_True(string language, bool isWebHostChannel, bool isJobHostChannel, bool expectedResult)
{
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher();
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
Assert.Equal(expectedResult, functionDispatcher.ShouldRestartWorkerChannel(language, isWebHostChannel, isJobHostChannel));
}
public async void FunctionDispatcherState_Default_NoFunctions()
{
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher();
Assert.Equal(FunctionDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
}
public void MaxProcessCount_ProcessCount_Set_Returns_ExpectedCount()
{
int expectedProcessCount = 3;
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString());
Assert.Equal(expectedProcessCount, functionDispatcher.MaxProcessCount);
}
public async Task FunctionDispatcher_ErroredWebHostChannel()
{
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher(throwOnProcessStartUp: true, addWebhostChannel: true);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.JavaLanguageWorkerName));
var testLogs = _testLogger.GetLogMessages();
Assert.False(testLogs.Any(m => m.FormattedMessage.Contains("Removing errored webhost language worker channel for runtime")));
}
public async void Starting_MultipleJobhostChannels_Succeeds()
{
int expectedProcessCount = 3;
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
var finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Assert.Equal(expectedProcessCount, finalChannelCount);
}
private async Task <int> WaitForJobhostWorkerChannelsToStartup(FunctionDispatcher functionDispatcher, int expectedCount)
{
int currentChannelCount = 0;
await TestHelpers.Await(() =>
{
currentChannelCount = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count();
return(currentChannelCount == expectedCount);
}, pollingInterval : 5 * 1000, timeout : 60 * 1000);
return(currentChannelCount);
}
public void MaxProcessCount_Returns_Default()
{
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher();
Assert.Equal(1, functionDispatcher.MaxProcessCount);
functionDispatcher = GetTestFunctionDispatcher("0");
Assert.Equal(1, functionDispatcher.MaxProcessCount);
functionDispatcher = GetTestFunctionDispatcher("-1");
Assert.Equal(1, functionDispatcher.MaxProcessCount);
}
public async void ShutdownChannels_NoFunctions()
{
var mockLanguageWorkerChannelManager = new Mock <ILanguageWorkerChannelManager>();
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(mockLanguageWorkerChannelManager: mockLanguageWorkerChannelManager);
Assert.Equal(functionDispatcher.State, FunctionDispatcherState.Default);
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
// Wait longer than debouce action.
await Task.Delay(6000);
mockLanguageWorkerChannelManager.Verify(m => m.ShutdownChannels(), Times.Once);
}
public async void Starting_MultipleJobhostChannels_Succeeds()
{
int expectedProcessCount = 3;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
var finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
Assert.Equal(finalChannelCount, expectedProcessCount);
// Verify LanguageWorkerChannelState when channel after it is initialized
Assert.True(functionDispatcher.WorkerState.GetChannels().All(ch => ch.State == LanguageWorkerChannelState.Initialized));
}
public static void IsSupported_Returns_False(string language, string funcMetadataLanguage)
{
var workerConfigs = TestHelpers.GetTestWorkerConfigs();
var eventManager = new Mock <IScriptEventManager>();
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = funcMetadataLanguage
};
FunctionDispatcher functionDispatcher = new FunctionDispatcher(eventManager.Object, new TestRpcServer(), null, workerConfigs, language);
Assert.False(functionDispatcher.IsSupported(func1));
}
public async void Starting_MultipleWebhostChannels_Succeeds()
{
int expectedProcessCount = 2;
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher(expectedProcessCount.ToString(), true);
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.JavaLanguageWorkerName));
var finalWebhostChannelCount = await WaitForWebhostWorkerChannelsToStartup(functionDispatcher.WebHostLanguageWorkerChannelManager, expectedProcessCount, "java");
Assert.Equal(expectedProcessCount, finalWebhostChannelCount);
var finalJobhostChannelCount = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count();
Assert.Equal(0, finalJobhostChannelCount);
}
public async Task FunctionDispatcher_DoNot_Restart_ErroredChannels_If_WorkerRuntime_DoesNotMatch()
{
int expectedProcessCount = 1;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
_javaTestChannel.RaiseWorkerError();
var testLogs = _testLogger.GetLogMessages();
Assert.False(testLogs.Any(m => m.FormattedMessage.Contains("Restarting worker channel for runtime:java")));
Assert.Equal(expectedProcessCount, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async Task ShutdownChannels_NoFunctions()
{
var mockLanguageWorkerChannelManager = new Mock <IWebHostLanguageWorkerChannelManager>();
mockLanguageWorkerChannelManager.Setup(m => m.ShutdownChannelsAsync()).Returns(Task.CompletedTask);
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher(mockwebHostLanguageWorkerChannelManager: mockLanguageWorkerChannelManager);
Assert.Equal(FunctionDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(new List <FunctionMetadata>());
// Wait longer than debouce action.
await Task.Delay(6000);
mockLanguageWorkerChannelManager.Verify(m => m.ShutdownChannelsAsync(), Times.Once);
}
private async Task <int> WaitForJobhostWorkerChannelsToStartup(FunctionDispatcher functionDispatcher, int expectedCount)
{
int currentChannelCount = 0;
await TestHelpers.Await(() =>
{
currentChannelCount = functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count();
if (currentChannelCount == expectedCount)
{
return(functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().All(ch => ch.State == LanguageWorkerChannelState.Initialized));
}
return(false);
}, pollingInterval : expectedCount * 5 * 1000, timeout : 60 * 1000);
return(currentChannelCount);
}
public async void FunctionDispatcherState_Transitions_From_Starting_To_Initialized()
{
FunctionDispatcher functionDispatcher = GetTestFunctionDispatcher();
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "node"
};
var functions = new List <FunctionMetadata>()
{
func1
};
await functionDispatcher.InitializeAsync(functions);
Assert.True(functionDispatcher.State == FunctionDispatcherState.Initializing || functionDispatcher.State == FunctionDispatcherState.Initialized);
await WaitForFunctionDispactherStateInitialized(functionDispatcher);
}
public async Task FunctionDispatcher_Error_BeyondThreshold_BucketIsAtOne()
{
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher("1");
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 1; i < 10; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerErrorWithCustomTimestamp(DateTime.UtcNow.AddHours(i));
}
}
Assert.Equal(1, functionDispatcher.LanguageWorkerErrors.Count);
}
public async Task FunctionDispatcher_Error_WithinThreshold_BucketFills()
{
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher("1");
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, 1);
for (int i = 0; i < 3; ++i)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(3, functionDispatcher.LanguageWorkerErrors.Count);
}
public async void FunctionDispatcher_Restart_ErroredChannels_ExcceedsLimit()
{
int expectedProcessCount = 2;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
foreach (var channel in functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels())
{
TestLanguageWorkerChannel testWorkerChannel = channel as TestLanguageWorkerChannel;
testWorkerChannel.RaiseWorkerError();
}
}
Assert.Equal(0, functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().Count());
}
public async void FunctionDispatcher_Restart_ErroredChannels_Succeeds()
{
int expectedProcessCount = 2;
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(expectedProcessCount.ToString());
await functionDispatcher.InitializeAsync(GetTestFunctionsList(LanguageWorkerConstants.NodeLanguageWorkerName));
await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
int finalChannelCount = 0;
for (int restartCount = 0; restartCount < expectedProcessCount * 3; restartCount++)
{
TestLanguageWorkerChannel testWorkerChannel = (TestLanguageWorkerChannel)functionDispatcher.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault();
if (functionDispatcher.LanguageWorkerErrors.Count < (expectedProcessCount * 3) - 1)
{
testWorkerChannel.RaiseWorkerError();
}
finalChannelCount = await WaitForJobhostWorkerChannelsToStartup(functionDispatcher, expectedProcessCount);
}
Assert.Equal(expectedProcessCount, finalChannelCount);
}
public async void ShutdownChannels_DotNetFunctions()
{
FunctionMetadata func1 = new FunctionMetadata()
{
Name = "func1",
Language = "dotnet"
};
var functions = new List <FunctionMetadata>()
{
func1
};
var mockLanguageWorkerChannelManager = new Mock <IWebHostLanguageWorkerChannelManager>();
FunctionDispatcher functionDispatcher = (FunctionDispatcher)GetTestFunctionDispatcher(mockwebHostLanguageWorkerChannelManager: mockLanguageWorkerChannelManager);
Assert.Equal(FunctionDispatcherState.Default, functionDispatcher.State);
await functionDispatcher.InitializeAsync(functions);
// Wait longer than debouce action.
await Task.Delay(6000);
mockLanguageWorkerChannelManager.Verify(m => m.ShutdownChannels(), Times.Once);
}
private LanguageWorkerChannel GetCurrentJobHostWorkerChannel()
{
FunctionDispatcher fd = Fixture.JobHost.FunctionDispatcher as FunctionDispatcher;
return((LanguageWorkerChannel)fd.JobHostLanguageWorkerChannelManager.GetChannels().FirstOrDefault());
}
