public WebScriptHostExceptionHandler(ScriptHostManager manager)
{
if (manager == null)
{
throw new ArgumentNullException(nameof(manager));
}
_manager = manager;
}
public async Task UpdateFileAndRestart()
{
CancellationTokenSource cts = new CancellationTokenSource();
var fixture = new NodeEndToEndTests.TestFixture();
var blob1 = UpdateOutputName("testblob", "first", fixture);
await fixture.Host.StopAsync();
var config = fixture.Host.ScriptConfig;
ExceptionDispatchInfo exception = null;
using (var manager = new ScriptHostManager(config))
{
// Background task to run while the main thread is pumping events at RunAndBlock().
Thread t = new Thread(_ =>
{
// don't start until the manager is running
TestHelpers.Await(() => manager.State == ScriptHostState.Running).Wait();
try
{
// Wait for initial execution.
TestHelpers.Await(() =>
{
bool exists = blob1.Exists();
return exists;
}, timeout: 10 * 1000).Wait();
// This changes the bindings so that we now write to blob2
var blob2 = UpdateOutputName("first", "second", fixture);
// wait for newly executed
TestHelpers.Await(() =>
{
bool exists = blob2.Exists();
return exists;
}, timeout: 30 * 1000).Wait();
}
catch (Exception ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
}
cts.Cancel();
});
t.Start();
manager.RunAndBlock(cts.Token);
t.Join();
Assert.True(exception == null, exception?.SourceException?.ToString());
}
}
public static void Main(string[] args)
{
if (args == null)
{
throw new ArgumentNullException("args");
}
string rootPath = Environment.CurrentDirectory;
if (args.Length > 0)
{
rootPath = (string)args[0];
}
ScriptHostConfiguration config = new ScriptHostConfiguration()
{
RootScriptPath = rootPath
};
ScriptHostManager scriptHostManager = new ScriptHostManager(config);
scriptHostManager.RunAndBlock();
}
public async Task HostHealthMonitor_TriggersShutdown_WhenHostUnhealthy()
{
string functionDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Functions", Guid.NewGuid().ToString());
Directory.CreateDirectory(functionDir);
string logDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Logs", Guid.NewGuid().ToString());
JObject hostConfig = new JObject
{
{ "id", "123456" }
};
File.WriteAllText(Path.Combine(functionDir, ScriptConstants.HostMetadataFileName), hostConfig.ToString());
var config = new ScriptHostConfiguration
{
RootScriptPath = functionDir,
RootLogPath = logDir,
FileLoggingMode = FileLoggingMode.Always,
};
// configure the monitor so it will fail within a couple seconds
config.HostHealthMonitor.HealthCheckInterval = TimeSpan.FromMilliseconds(100);
config.HostHealthMonitor.HealthCheckWindow = TimeSpan.FromSeconds(1);
config.HostHealthMonitor.HealthCheckThreshold = 5;
var environmentMock = new Mock <IScriptHostEnvironment>(MockBehavior.Strict);
environmentMock.Setup(p => p.Shutdown());
var mockSettings = new Mock <ScriptSettingsManager>(null);
mockSettings.Setup(p => p.IsAppServiceEnvironment).Returns(true);
var eventManagerMock = new Mock <IScriptEventManager>();
var hostHealthConfig = new HostHealthMonitorConfiguration();
var mockHostPerformanceManager = new Mock <HostPerformanceManager>(mockSettings.Object, hostHealthConfig);
bool underHighLoad = false;
mockHostPerformanceManager.Setup(p => p.IsUnderHighLoad(It.IsAny <Collection <string> >(), It.IsAny <ILogger>()))
.Callback <Collection <string>, ILogger>((c, l) =>
{
c.Add("Connections");
})
.Returns(() => underHighLoad);
var loggerProvider = new TestLoggerProvider();
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
var hostManager = new ScriptHostManager(config, mockSettings.Object, new ScriptHostFactory(), eventManagerMock.Object, environmentMock.Object, loggerProviderFactory, mockHostPerformanceManager.Object);
Assert.True(hostManager.ShouldMonitorHostHealth);
Task runTask = Task.Run(() => hostManager.RunAndBlock());
await TestHelpers.Await(() => hostManager.State == ScriptHostState.Running);
// now that host is running make host unhealthy and wait
// for host shutdown
underHighLoad = true;
await TestHelpers.Await(() => hostManager.State == ScriptHostState.Error);
Assert.Equal(ScriptHostState.Error, hostManager.State);
environmentMock.Verify(p => p.Shutdown(), Times.Once);
// we expect a few restart iterations
var thresholdErrors = loggerProvider.GetAllLogMessages().Where(p => p.Exception is InvalidOperationException && p.Exception.Message == "Host thresholds exceeded: [Connections]. For more information, see https://aka.ms/functions-thresholds.");
Assert.True(thresholdErrors.Count() > 1);
var log = loggerProvider.GetAllLogMessages().Last();
Assert.True(loggerProvider.GetAllLogMessages().Count(p => p.FormattedMessage == "Host is unhealthy. Initiating a restart." && p.Level == LogLevel.Error) > 0);
Assert.Equal("Host unhealthy count exceeds the threshold of 5 for time window 00:00:01. Initiating shutdown.", log.FormattedMessage);
Assert.Equal(LogLevel.Error, log.Level);
}
public async Task UpdateFileAndRestart()
{
var fixture = new NodeScriptHostTests.TestFixture(false);
var config = fixture.Host.ScriptConfig;
config.OnConfigurationApplied = c =>
{
c.Functions = new Collection <string> {
"TimerTrigger"
};
};
var blob1 = await UpdateOutputName("testblob", "first", fixture);
using (var eventManager = new ScriptEventManager())
using (var manager = new ScriptHostManager(config, eventManager))
{
string GetErrorTraces()
{
var messages = fixture.LoggerProvider.GetAllLogMessages()
.Where(t => t.Level == LogLevel.Error)
.Select(t => t.FormattedMessage);
return(string.Join(Environment.NewLine, messages));
}
List <Exception> exceptions = new List <Exception>();
// Background task to run while the main thread is pumping events at RunAndBlock().
Thread backgroundThread = new Thread(_ =>
{
try
{
// don't start until the manager is running
TestHelpers.Await(() => manager.State == ScriptHostState.Running,
userMessageCallback: GetErrorTraces).GetAwaiter().GetResult();
// Wait for initial execution.
TestHelpers.Await(async() =>
{
bool exists = await blob1.ExistsAsync();
return(exists);
}, timeout: 10 * 1000, userMessageCallback: GetErrorTraces).GetAwaiter().GetResult();
// This changes the bindings so that we now write to blob2
var blob2 = UpdateOutputName("first", "testblob", fixture).Result;
// wait for newly executed
TestHelpers.Await(async() =>
{
bool exists = await blob2.ExistsAsync();
return(exists);
}, timeout: 30 * 1000, userMessageCallback: GetErrorTraces).GetAwaiter().GetResult();
// The TimerTrigger can fire before the host is fully started. To be more
// reliably clean up the test, wait until it is running before calling Stop.
TestHelpers.Await(() => manager.State == ScriptHostState.Running,
userMessageCallback: GetErrorTraces).GetAwaiter().GetResult();
}
catch (Exception ex)
{
exceptions.Add(ex);
}
finally
{
try
{
// Calling Stop (rather than using a token) lets us wait until all listeners have stopped.
manager.Stop();
}
catch (Exception ex)
{
exceptions.Add(ex);
}
}
});
try
{
backgroundThread.Start();
manager.RunAndBlock();
Assert.True(backgroundThread.Join(60000), "The background task did not complete in 60 seconds.");
string exceptionString = string.Join(Environment.NewLine, exceptions.Select(p => p.ToString()));
Assert.True(exceptions.Count() == 0, exceptionString);
}
finally
{
// make sure to put the original names back
await UpdateOutputName("first", "testblob", fixture);
}
}
}
public async Task RenameFunctionAndRestart()
{
var oldDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestScripts/Node/TimerTrigger");
var newDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestScripts/Node/MovedTrigger");
var fixture = new NodeScriptHostTests.TestFixture(false);
var config = fixture.Host.ScriptConfig;
config.OnConfigurationApplied = c =>
{
c.Functions = new Collection <string> {
"TimerTrigger", "MovedTrigger"
};
};
var blob = fixture.TestOutputContainer.GetBlockBlobReference("testblob");
await blob.DeleteIfExistsAsync();
var mockEnvironment = new Mock <IScriptHostEnvironment>();
using (var eventManager = new ScriptEventManager())
using (var manager = new ScriptHostManager(config, eventManager, mockEnvironment.Object))
using (var resetEvent = new ManualResetEventSlim())
{
List <Exception> exceptions = new List <Exception>();
mockEnvironment.Setup(e => e.RestartHost())
.Callback(() =>
{
resetEvent.Set();
manager.RestartHost();
});
// Background task to run while the main thread is pumping events at RunAndBlock().
Thread backgroundThread = new Thread(_ =>
{
try
{
// don't start until the manager is running
TestHelpers.Await(() => manager.State == ScriptHostState.Running,
userMessageCallback: () => "Host did not start in time.").GetAwaiter().GetResult();
// Wait for initial execution.
TestHelpers.Await(async() =>
{
bool exists = await blob.ExistsAsync();
return(exists);
}, timeout: 10 * 1000,
userMessageCallback: () => $"Blob '{blob.Uri}' was not created by 'TimerTrigger' in time.").GetAwaiter().GetResult();
// find __dirname from blob
string text;
using (var stream = new MemoryStream())
{
blob.DownloadToStreamAsync(stream).Wait();
text = System.Text.Encoding.UTF8.GetString(stream.ToArray());
}
Assert.Contains("TimerTrigger", text);
// rename directory & delete old blob
Directory.Move(oldDirectory, newDirectory);
resetEvent.Wait(TimeSpan.FromSeconds(10));
blob.DeleteIfExistsAsync().GetAwaiter().GetResult();
// wait for newly executed
TestHelpers.Await(async() =>
{
bool exists = await blob.ExistsAsync();
return(exists);
}, timeout: 30 * 1000,
userMessageCallback: () => $"Blob '{blob.Uri}' was not created by 'MovedTrigger' in time.").GetAwaiter().GetResult();
using (var stream = new MemoryStream())
{
blob.DownloadToStreamAsync(stream).Wait();
text = System.Text.Encoding.UTF8.GetString(stream.ToArray());
}
Assert.Contains("MovedTrigger", text);
// The TimerTrigger can fire before the host is fully started. To be more
// reliably clean up the test, wait until it is running before calling Stop.
TestHelpers.Await(() => manager.State == ScriptHostState.Running).GetAwaiter().GetResult();
}
catch (Exception ex)
{
exceptions.Add(ex);
}
finally
{
try
{
manager.Stop();
}
catch (Exception ex)
{
exceptions.Add(ex);
}
}
});
try
{
backgroundThread.Start();
manager.RunAndBlock();
Assert.True(backgroundThread.Join(60000), "The background task did not complete in 60 seconds.");
string exceptionString = string.Join(Environment.NewLine, exceptions.Select(p => p.ToString()));
Assert.True(exceptions.Count() == 0, exceptionString);
}
finally
{
// Move the directory back after the host has stopped to prevent
// unnecessary host restarts
if (Directory.Exists(newDirectory))
{
Directory.Move(newDirectory, oldDirectory);
}
}
}
}
public async Task RunAndBlock_SetsLastError_WhenExceptionIsThrown()
{
ScriptHostConfiguration config = new ScriptHostConfiguration()
{
RootScriptPath = @"TestScripts\Empty"
};
var factoryMock = new Mock<IScriptHostFactory>();
var scriptHostFactory = new TestScriptHostFactory()
{
Throw = true
};
var hostManager = new ScriptHostManager(config, _settingsManager, scriptHostFactory);
Task taskIgnore = Task.Run(() => hostManager.RunAndBlock());
// we expect a host exception immediately
await Task.Delay(2000);
Assert.Equal(ScriptHostState.Error, hostManager.State);
Assert.False(hostManager.CanInvoke());
Assert.NotNull(hostManager.LastError);
Assert.Equal("Kaboom!", hostManager.LastError.Message);
// now verify that if no error is thrown on the next iteration
// the cached error is cleared
scriptHostFactory.Throw = false;
await TestHelpers.Await(() =>
{
return hostManager.State == ScriptHostState.Running;
});
Assert.Null(hostManager.LastError);
Assert.True(hostManager.CanInvoke());
Assert.Equal(ScriptHostState.Running, hostManager.State);
}
public async Task EmptyHost_StartsSuccessfully()
{
string functionDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Functions", Guid.NewGuid().ToString());
Directory.CreateDirectory(functionDir);
// important for the repro that this directory does not exist
string logDir = Path.Combine(TestHelpers.FunctionsTestDirectory, "Logs", Guid.NewGuid().ToString());
JObject hostConfig = new JObject
{
{ "id", "123456" }
};
File.WriteAllText(Path.Combine(functionDir, ScriptConstants.HostMetadataFileName), hostConfig.ToString());
ScriptHostConfiguration config = new ScriptHostConfiguration
{
RootScriptPath = functionDir,
RootLogPath = logDir,
FileLoggingMode = FileLoggingMode.Always
};
ScriptHostManager hostManager = new ScriptHostManager(config);
Task runTask = Task.Run(() => hostManager.RunAndBlock());
await TestHelpers.Await(() => hostManager.State == ScriptHostState.Running, timeout: 10000);
hostManager.Stop();
Assert.Equal(ScriptHostState.Default, hostManager.State);
await Task.Delay(FileTraceWriter.LogFlushIntervalMs);
string hostLogFilePath = Directory.EnumerateFiles(Path.Combine(logDir, "Host")).Single();
string hostLogs = File.ReadAllText(hostLogFilePath);
Assert.Contains("Generating 0 job function(s)", hostLogs);
Assert.Contains("No job functions found.", hostLogs);
Assert.Contains("Job host started", hostLogs);
Assert.Contains("Job host stopped", hostLogs);
}
private MockExceptionHandler GetExceptionHandler(ScriptHostManager manager)
{
return(manager.Instance.ScriptConfig.HostConfig.GetService <IWebJobsExceptionHandler>() as MockExceptionHandler);
}
public async Task UpdateFileAndRestart()
{
CancellationTokenSource cts = new CancellationTokenSource();
var fixture = new NodeEndToEndTests.TestFixture();
var blob1 = UpdateOutputName("testblob", "first", fixture);
await fixture.Host.StopAsync();
var config = fixture.Host.ScriptConfig;
ExceptionDispatchInfo exception = null;
using (var eventManager = new ScriptEventManager())
using (var manager = new ScriptHostManager(config, eventManager))
{
// Background task to run while the main thread is pumping events at RunAndBlock().
Thread t = new Thread(_ =>
{
// don't start until the manager is running
TestHelpers.Await(() => manager.State == ScriptHostState.Running).Wait();
try
{
// Wait for initial execution.
TestHelpers.Await(() =>
{
bool exists = blob1.Exists();
return(exists);
}, timeout: 10 * 1000).Wait();
// This changes the bindings so that we now write to blob2
var blob2 = UpdateOutputName("first", "testblob", fixture);
// wait for newly executed
TestHelpers.Await(() =>
{
bool exists = blob2.Exists();
return(exists);
}, timeout: 30 * 1000).Wait();
}
catch (Exception ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
}
finally
{
try
{
UpdateOutputName("first", "testblob", fixture);
}
catch
{
}
}
cts.Cancel();
});
t.Start();
manager.RunAndBlock(cts.Token);
t.Join();
Assert.True(exception == null, exception?.SourceException?.ToString());
}
}
public void Restart_CreatesNew_FunctionTraceWriter()
{
string functionDir = @"TestScripts\CSharp";
var traceWriter = new TestTraceWriter(System.Diagnostics.TraceLevel.Info);
ScriptHostConfiguration config = new ScriptHostConfiguration
{
RootScriptPath = functionDir,
FileLoggingMode = FileLoggingMode.Always,
TraceWriter = traceWriter
};
string hostJsonPath = Path.Combine(functionDir, ScriptConstants.HostMetadataFileName);
string originalHostJson = File.ReadAllText(hostJsonPath);
// Only load two functions to start:
JObject hostConfig = new JObject
{
{ "id", "123456" },
{ "functions", new JArray("ManualTrigger", "Scenarios") }
};
File.WriteAllText(hostJsonPath, hostConfig.ToString());
CancellationTokenSource cts = new CancellationTokenSource();
ExceptionDispatchInfo exception = null;
try
{
using (var manager = new ScriptHostManager(config))
{
// Background task to run while the main thread is pumping events at RunAndBlock().
Thread t = new Thread(_ =>
{
try
{
// don't start until the manager is running
TestHelpers.Await(() => manager.State == ScriptHostState.Running).Wait();
var firstFileWriters = GetRemovableTraceWriters(manager.Instance);
Assert.Equal(2, firstFileWriters.Count());
// update the host.json to only have one function
hostConfig["functions"] = new JArray("ManualTrigger");
traceWriter.Traces.Clear();
File.WriteAllText(hostJsonPath, hostConfig.ToString());
TestHelpers.Await(() => traceWriter.Traces.Select(p => p.Message).Contains("Job host started")).Wait();
TestHelpers.Await(() => manager.State == ScriptHostState.Running).Wait();
var secondFileWriters = GetRemovableTraceWriters(manager.Instance);
Assert.Equal(1, secondFileWriters.Count());
// make sure we have a new instance of the ManualTrigger writer and that it does
// not throw an ObjectDisposedException when we use it
Assert.DoesNotContain(secondFileWriters.Single(), firstFileWriters);
secondFileWriters.Single().Info("test");
// add back the other function -- make sure the writer is not disposed
hostConfig["functions"] = new JArray("ManualTrigger", "Scenarios");
traceWriter.Traces.Clear();
File.WriteAllText(hostJsonPath, hostConfig.ToString());
TestHelpers.Await(() => traceWriter.Traces.Select(p => p.Message).Contains("Job host started")).Wait();
TestHelpers.Await(() => manager.State == ScriptHostState.Running).Wait();
var thirdFileWriters = GetRemovableTraceWriters(manager.Instance);
Assert.Equal(2, thirdFileWriters.Count());
// make sure these are all new and that they also do not throw
var previousWriters = firstFileWriters.Concat(secondFileWriters);
Assert.DoesNotContain(thirdFileWriters.First(), previousWriters);
Assert.DoesNotContain(thirdFileWriters.Last(), previousWriters);
thirdFileWriters.First().Info("test");
thirdFileWriters.Last().Info("test");
}
catch (Exception ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
}
finally
{
cts.Cancel();
}
});
t.Start();
manager.RunAndBlock(cts.Token);
t.Join();
}
Assert.True(exception == null, exception?.SourceException?.ToString());
}
finally
{
File.WriteAllText(hostJsonPath, originalHostJson);
}
}
public async Task RenameFunctionAndRestart()
{
var oldDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestScripts/Node/TimerTrigger");
var newDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestScripts/Node/MovedTrigger");
CancellationTokenSource cts = new CancellationTokenSource();
var fixture = new NodeEndToEndTests.TestFixture();
await fixture.Host.StopAsync();
var config = fixture.Host.ScriptConfig;
var blob = fixture.TestOutputContainer.GetBlockBlobReference("testblob");
ExceptionDispatchInfo exception = null;
var mockEnvironment = new Mock <IScriptHostEnvironment>();
using (var eventManager = new ScriptEventManager())
using (var manager = new ScriptHostManager(config, eventManager, mockEnvironment.Object))
using (var resetEvent = new ManualResetEventSlim())
{
mockEnvironment.Setup(e => e.RestartHost())
.Callback(() =>
{
resetEvent.Set();
manager.RestartHost();
});
// Background task to run while the main thread is pumping events at RunAndBlock().
Thread t = new Thread(_ =>
{
// don't start until the manager is running
TestHelpers.Await(() => manager.State == ScriptHostState.Running).Wait();
try
{
// Wait for initial execution.
TestHelpers.Await(() =>
{
bool exists = blob.Exists();
return(exists);
}, timeout: 10 * 1000).Wait();
// find __dirname from blob
string text;
using (var stream = new MemoryStream())
{
blob.DownloadToStream(stream);
text = System.Text.Encoding.UTF8.GetString(stream.ToArray());
}
Assert.Contains("TimerTrigger", text);
// rename directory & delete old blob
Directory.Move(oldDirectory, newDirectory);
resetEvent.Wait(TimeSpan.FromSeconds(10));
blob.Delete();
// wait for newly executed
TestHelpers.Await(() =>
{
bool exists = blob.Exists();
return(exists);
}, timeout: 30 * 1000).Wait();
using (var stream = new MemoryStream())
{
blob.DownloadToStream(stream);
text = System.Text.Encoding.UTF8.GetString(stream.ToArray());
}
Assert.Contains("MovedTrigger", text);
}
catch (Exception ex)
{
exception = ExceptionDispatchInfo.Capture(ex);
}
finally
{
try
{
Directory.Move(newDirectory, oldDirectory);
}
catch
{
}
}
cts.Cancel();
});
t.Start();
manager.RunAndBlock(cts.Token);
t.Join();
Assert.True(exception == null, exception?.SourceException?.ToString());
}
}
private MockExceptionHandler GetExceptionHandler(ScriptHostManager manager)
{
return manager.Instance.ScriptConfig.HostConfig.GetService<IWebJobsExceptionHandler>() as MockExceptionHandler;
}
