public async Task OnTimeoutExceptionAsync(ExceptionDispatchInfo exceptionInfo, TimeSpan timeoutGracePeriod)
{
FunctionTimeoutException timeoutException = exceptionInfo.SourceException as FunctionTimeoutException;
if (timeoutException?.Task != null)
{
// We may double the timeoutGracePeriod here by first waiting to see if the iniital
// function task that started the exception has completed.
Task completedTask = await Task.WhenAny(timeoutException.Task, Task.Delay(timeoutGracePeriod));
// If the function task has completed, simply return. The host has already logged the timeout.
if (completedTask == timeoutException.Task)
{
return;
}
}
LogErrorAndFlush("A function timeout has occurred. Host is shutting down.", exceptionInfo.SourceException);
// We can't wait on this as it may cause a deadlock if the timeout was fired
// by a Listener that cannot stop until it has completed.
Task ignoreTask = _manager.StopAsync();
// Give the manager and all running tasks some time to shut down gracefully.
await Task.Delay(timeoutGracePeriod);
HostingEnvironment.InitiateShutdown();
}
private async Task RunTimeoutTest(string scriptLang, string functionName)
{
TestHelpers.ClearFunctionLogs(functionName);
TimeSpan testTimeout = TimeSpan.FromSeconds(3);
var traceWriter = new TestTraceWriter(TraceLevel.Verbose);
using (var manager = await CreateAndStartScriptHostManager(scriptLang, functionName, testTimeout, traceWriter))
{
string testData = Guid.NewGuid().ToString();
Dictionary <string, object> arguments = new Dictionary <string, object>
{
{ "inputData", testData },
};
FunctionTimeoutException ex = await Assert.ThrowsAsync <FunctionTimeoutException>(() => manager.Instance.CallAsync(functionName, arguments));
await TestHelpers.Await(() =>
{
// make sure logging from within the function worked
// TODO: This doesn't appear to work for Powershell in AppVeyor. Need to investigate.
//bool hasTestData = inProgressLogs.Any(l => l.Contains(testData));
var expectedMessage = $"Timeout value of {testTimeout} was exceeded by function: Functions.{functionName}";
var traces = string.Join(Environment.NewLine, traceWriter.Traces);
return(traces.Contains(expectedMessage));
});
var exception = GetExceptionHandler(manager).TimeoutExceptionInfos.Single().SourceException;
Assert.IsType <FunctionTimeoutException>(exception);
}
}
public async Task OnTimeoutExceptionAsync(ExceptionDispatchInfo exceptionInfo, TimeSpan timeoutGracePeriod)
{
FunctionTimeoutException timeoutException = exceptionInfo.SourceException as FunctionTimeoutException;
if (timeoutException?.Task != null)
{
// We may double the timeoutGracePeriod here by first waiting to see if the iniital
// function task that started the exception has completed.
Task completedTask = await Task.WhenAny(timeoutException.Task, Task.Delay(timeoutGracePeriod));
// If the function task has completed, simply return. The host has already logged the timeout.
if (completedTask == timeoutException.Task)
{
return;
}
}
LogErrorAndFlush("A function timeout has occurred. Host is shutting down.", exceptionInfo.SourceException);
// We can't wait on this as it may cause a deadlock if the timeout was fired
// by a Listener that cannot stop until it has completed.
// TODO: DI (FACAVAL) The shutdown call will invoke the host stop... but we may need to do this
// explicitly in order to pass the timeout.
// Task ignoreTask = _hostManager.StopAsync();
// Give the manager and all running tasks some time to shut down gracefully.
//await Task.Delay(timeoutGracePeriod);
_applicationLifetime.StopApplication();
}
private async Task RunTimeoutTest(string scriptLang, string functionName)
{
TestHelpers.ClearFunctionLogs(functionName);
TimeSpan testTimeout = TimeSpan.FromSeconds(3);
IHostBuilder builder = CreateTimeoutHostBuilder($@"TestScripts\{scriptLang}", TimeSpan.FromSeconds(3), functionName);
using (var host = builder.Build())
{
await host.StartAsync();
ScriptHost scriptHost = host.GetScriptHost();
string testData = Guid.NewGuid().ToString();
Dictionary <string, object> arguments = new Dictionary <string, object>
{
{ "inputData", testData },
};
FunctionTimeoutException ex = await Assert.ThrowsAsync <FunctionTimeoutException>(() => scriptHost.CallAsync(functionName, arguments));
await TestHelpers.Await(() =>
{
// make sure logging from within the function worked
// TODO: This doesn't appear to work for Powershell in AppVeyor. Need to investigate.
// bool hasTestData = inProgressLogs.Any(l => l.Contains(testData));
var expectedMessage = $"Timeout value of {testTimeout} exceeded by function 'Functions.{functionName}'";
var traces = string.Join(Environment.NewLine, _loggerProvider.GetAllLogMessages().Where(t => t.FormattedMessage != null).Select(p => p.FormattedMessage));
return(traces.Contains(expectedMessage));
});
var exception = GetExceptionHandler(host).TimeoutExceptionInfos.Single().SourceException;
Assert.IsType <FunctionTimeoutException>(exception);
}
}
private async Task RunTokenTest(string scenario, Action <IEnumerable <string> > verify)
{
string functionName = "TimeoutToken";
TestHelpers.ClearFunctionLogs(functionName);
IHostBuilder builder = CreateTimeoutHostBuilder(@"TestScripts\CSharp", TimeSpan.FromSeconds(3), functionName);
using (var host = builder.Build())
{
await host.StartAsync();
var scriptHost = host.GetScriptHost();
Dictionary <string, object> arguments = new Dictionary <string, object>
{
{ "input", scenario },
};
FunctionTimeoutException ex = await Assert.ThrowsAsync <FunctionTimeoutException>(() => scriptHost.CallAsync(functionName, arguments));
var exception = GetExceptionHandler(host).TimeoutExceptionInfos.Single().SourceException;
Assert.IsType <FunctionTimeoutException>(exception);
verify(_loggerProvider.GetAllLogMessages().Where(t => t.FormattedMessage != null).Select(t => t.FormattedMessage));
}
}
private async Task RunTokenTest(string scenario, Action <IEnumerable <string> > verify)
{
string functionName = "TimeoutToken";
TestHelpers.ClearFunctionLogs(functionName);
using (var manager = await CreateAndStartScriptHostManager("CSharp", functionName, TimeSpan.FromSeconds(3)))
{
Dictionary <string, object> arguments = new Dictionary <string, object>
{
{ "input", scenario },
};
FunctionTimeoutException ex = await Assert.ThrowsAsync <FunctionTimeoutException>(() => manager.Instance.CallAsync(functionName, arguments));
var exception = GetExceptionHandler(manager).TimeoutExceptionInfos.Single().SourceException;
Assert.IsType <FunctionTimeoutException>(exception);
verify(_loggerProvider.GetAllLogMessages().Where(t => t.FormattedMessage != null).Select(t => t.FormattedMessage));
}
}
public async Task OnTimeoutExceptionAsync(ExceptionDispatchInfo exceptionInfo, TimeSpan timeoutGracePeriod)
{
FunctionTimeoutException timeoutException = exceptionInfo.SourceException as FunctionTimeoutException;
if (timeoutException?.Task != null)
{
// We may double the timeoutGracePeriod here by first waiting to see if the iniital
// function task that started the exception has completed.
Task completedTask = await Task.WhenAny(timeoutException.Task, Task.Delay(timeoutGracePeriod));
// If the function task has completed, simply return. The host has already logged the timeout.
if (completedTask == timeoutException.Task)
{
return;
}
}
// We can't wait on this as it may cause a deadlock if the timeout was fired
// by a Listener that cannot stop until it has completed.
// TODO: DI (FACAVAL) The shutdown call will invoke the host stop... but we may need to do this
// explicitly in order to pass the timeout.
// Task ignoreTask = _hostManager.StopAsync();
// Give the manager and all running tasks some time to shut down gracefully.
//await Task.Delay(timeoutGracePeriod);
IFunctionInvocationDispatcher functionInvocationDispatcher = _functionInvocationDispatcherFactory.GetFunctionDispatcher();
if (!functionInvocationDispatcher.State.Equals(FunctionInvocationDispatcherState.Default))
{
_logger.LogWarning($"A function timeout has occurred. Restarting worker process executing invocationId '{timeoutException.InstanceId}'.", exceptionInfo.SourceException);
// If invocation id is not found in any of the workers => worker is already disposed. No action needed.
await functionInvocationDispatcher.RestartWorkerWithInvocationIdAsync(timeoutException.InstanceId.ToString());
_logger.LogWarning("Restart of language worker process(es) completed.", exceptionInfo.SourceException);
}
else
{
LogErrorAndFlush("A function timeout has occurred. Host is shutting down.", exceptionInfo.SourceException);
_applicationLifetime.StopApplication();
}
}
private async Task RunTokenTest(string scenario, Action <IEnumerable <string> > verify)
{
string functionName = "TimeoutToken";
TestHelpers.ClearFunctionLogs(functionName);
var traceWriter = new TestTraceWriter(TraceLevel.Info);
using (var manager = await CreateAndStartScriptHostManager("CSharp", functionName, TimeSpan.FromSeconds(3), traceWriter))
{
Dictionary <string, object> arguments = new Dictionary <string, object>
{
{ "input", scenario },
};
FunctionTimeoutException ex = await Assert.ThrowsAsync <FunctionTimeoutException>(() => manager.Instance.CallAsync(functionName, arguments));
var exception = GetExceptionHandler(manager).TimeoutExceptionInfos.Single().SourceException;
Assert.IsType <FunctionTimeoutException>(exception);
verify(traceWriter.Traces.Select(t => t.ToString()));
}
}
