public void ReconnectedClearsTimedOutAndHasBeenWarnedFlags()
{
var mockTransport = new Mock <IClientTransport>();
mockTransport.Setup(t => t.SupportsKeepAlive).Returns(true);
var mockConnection = new Mock <IConnection>();
mockConnection.Setup(c => c.Transport).Returns(mockTransport.Object);
mockConnection.Setup(c => c.KeepAliveData).Returns(new KeepAliveData(new TimeSpan(0, 0, 9)));
mockConnection.Setup(c => c.State).Returns(ConnectionState.Connected);
using (var monitor = new HeartbeatMonitor(mockConnection.Object, new object(), new TimeSpan(1, 0, 0)))
{
monitor.Start();
// sets TimedOut flag
monitor.Beat(new TimeSpan(0, 10, 0));
// sets HasBeenWarned flag
monitor.Beat(new TimeSpan(0, 0, 7));
Assert.True(monitor.TimedOut);
Assert.True(monitor.HasBeenWarned);
monitor.Reconnected();
Assert.False(monitor.TimedOut);
Assert.False(monitor.HasBeenWarned);
}
}
public void NormalConnectionTest()
{
// Arrange
var connection = new Mock <Client.IConnection>();
var transport = new Mock <IClientTransport>();
// Setting the values such that a timeout or timeout warning isn't issued
var keepAliveData = new KeepAliveData(
timeoutWarning: TimeSpan.FromSeconds(5),
timeout: TimeSpan.FromSeconds(10),
checkInterval: TimeSpan.FromSeconds(2)
);
using (var monitor = new HeartbeatMonitor(connection.Object, new object(), keepAliveData.CheckInterval))
{
connection.Setup(m => m.LastMessageAt).Returns(DateTime.UtcNow);
connection.Setup(m => m.KeepAliveData).Returns(keepAliveData);
connection.Setup(m => m.State).Returns(ConnectionState.Connected);
connection.Setup(m => m.Transport).Returns(transport.Object);
monitor.Start();
// Act - Setting timespan to be less than timeout and timeout warning
monitor.Beat(TimeSpan.FromSeconds(2));
// Assert
Assert.False(monitor.TimedOut);
Assert.False(monitor.HasBeenWarned);
}
}
public static void Run(bool console = false)
{
BackgroundPool.config = new SettingsConfiguration();
oUsToMonitor = config.OUsToMonitor;
oUsDNToMonitor = config.OUsDNToMonitor;
try
{
InitializeLogging(console);
PerformStartupChecks(config);
hb = new HeartbeatMonitor();
hb.configuration = config;
hb.Start();
threadBackgroundWorker.IsBackground = true;
if (threadBackgroundWorker.ThreadState.HasFlag(System.Threading.ThreadState.Unstarted))
{
threadBackgroundWorker.Start();
}
}
catch (Exception e)
{
if (e.InnerException != null)
{
Console.Error.WriteLine(e.InnerException.Message);
log.LogFatal(e.InnerException.Message);
}
Console.Error.WriteLine(e.Message);
log.LogFatal(e.Message);
throw;
}
}
public void ConnectionTimeoutTest()
{
// Arrange
var connection = new Mock<Client.IConnection>();
var transport = new Mock<IClientTransport>();
transport.Setup(m => m.SupportsKeepAlive).Returns(true);
// Setting the values such that a timeout happens almost instantly
var keepAliveData = new KeepAliveData(
timeoutWarning: TimeSpan.FromSeconds(10),
timeout: TimeSpan.FromSeconds(1),
checkInterval: TimeSpan.FromSeconds(2)
);
using (var monitor = new HeartbeatMonitor(connection.Object, new object(), keepAliveData.CheckInterval))
{
connection.Setup(m => m.LastMessageAt).Returns(DateTime.UtcNow);
connection.Setup(m => m.KeepAliveData).Returns(keepAliveData);
connection.Setup(m => m.State).Returns(ConnectionState.Connected);
connection.Setup(m => m.Transport).Returns(transport.Object);
monitor.Start();
// Act - Setting timespan to be greater then timeout
monitor.Beat(TimeSpan.FromSeconds(5));
// Assert
Assert.True(monitor.TimedOut);
Assert.False(monitor.HasBeenWarned);
transport.Verify(m => m.LostConnection(connection.Object), Times.Once());
}
}
public void ConnectionTimeoutTest()
{
// Arrange
var connection = new Mock <Client.IConnection>();
var transport = new Mock <IClientTransport>();
transport.Setup(m => m.SupportsKeepAlive).Returns(true);
// Setting the values such that a timeout happens almost instantly
var keepAliveData = new KeepAliveData(
timeoutWarning: TimeSpan.FromSeconds(10),
timeout: TimeSpan.FromSeconds(1),
checkInterval: TimeSpan.FromSeconds(2)
);
using (var monitor = new HeartbeatMonitor(connection.Object, new object(), keepAliveData.CheckInterval))
{
connection.Setup(m => m.LastMessageAt).Returns(DateTime.UtcNow);
connection.Setup(m => m.KeepAliveData).Returns(keepAliveData);
connection.Setup(m => m.State).Returns(ConnectionState.Connected);
connection.Setup(m => m.Transport).Returns(transport.Object);
monitor.Start();
// Act - Setting timespan to be greater then timeout
monitor.Beat(TimeSpan.FromSeconds(5));
// Assert
Assert.True(monitor.TimedOut);
Assert.False(monitor.HasBeenWarned);
transport.Verify(m => m.LostConnection(connection.Object), Times.Once());
}
}
public void TestSuccess()
{
long actualNumHeartbeats = 0;
_heartbeat.Setup(x => x.Beat())
.Returns(() => Task.Factory.StartNew(() => { Interlocked.Increment(ref actualNumHeartbeats); }));
HeartbeatMonitor monitor;
using (monitor = new HeartbeatMonitor(_heartbeat.Object,
Debugger.Instance,
TimeSpan.FromSeconds(value: 0.1),
failureThreshold: 1,
enabledWithAttachedDebugger: true,
useHeartbeatFailureDetection: true,
allowRemoteHeartbeatDisable: true,
connectionId: _connectionId,
endPointName: "Test",
locEndPoint: _localEndPoint,
remoteEndPoint: _remoteEndPoint))
{
var failureDetected = false;
monitor.OnFailure += unuse => failureDetected = true;
monitor.Start();
Thread.Sleep(TimeSpan.FromSeconds(value: 1));
Console.WriteLine("# heartbeats: {0}", monitor.NumHeartbeats);
monitor.FailureDetected.Should().BeFalse();
failureDetected.Should().BeFalse();
}
// There should be 10 heartbeats in a perfect world, let's just verify that we've got half of that
monitor.NumHeartbeats.Should().BeGreaterOrEqualTo(expected: 5);
}
public void ReconnectedClearsTimedOutAndHasBeenWarnedFlags()
{
var mockTransport = new Mock<IClientTransport>();
mockTransport.Setup(t => t.SupportsKeepAlive).Returns(true);
var mockConnection = new Mock<IConnection>();
mockConnection.Setup(c => c.Transport).Returns(mockTransport.Object);
mockConnection.Setup(c => c.KeepAliveData).Returns(new KeepAliveData(new TimeSpan(0, 0, 9)));
mockConnection.Setup(c => c.State).Returns(ConnectionState.Connected);
using (var monitor = new HeartbeatMonitor(mockConnection.Object, new object(), new TimeSpan(1, 0, 0)))
{
monitor.Start();
// sets TimedOut flag
monitor.Beat(new TimeSpan(0, 10, 0));
// sets HasBeenWarned flag
monitor.Beat(new TimeSpan(0, 0, 7));
Assert.True(monitor.TimedOut);
Assert.True(monitor.HasBeenWarned);
monitor.Reconnected();
Assert.False(monitor.TimedOut);
Assert.False(monitor.HasBeenWarned);
}
}
public void TestDetectFailure4()
{
const bool enabledWithAttachedDebugger = false;
_debugger.Setup(x => x.IsDebuggerAttached).Returns(value: true);
long actualNumHeartbeats = 0;
DateTime?failureStarted = null;
_heartbeat.Setup(x => x.Beat())
.Returns(() => Task.Factory.StartNew(() =>
{
// Let's simulate failure by blocking the task
if (++actualNumHeartbeats == 25)
{
failureStarted = DateTime.Now;
Thread.Sleep(millisecondsTimeout: 10000);
}
}));
HeartbeatMonitor monitor;
using (
monitor =
new HeartbeatMonitor(_heartbeat.Object,
_debugger.Object,
TimeSpan.FromSeconds(value: 0.01),
failureThreshold: 1,
enabledWithAttachedDebugger: enabledWithAttachedDebugger,
useHeartbeatFailureDetection: true,
allowRemoteHeartbeatDisable: true,
connectionId: _connectionId,
endPointName: "Test",
locEndPoint: _localEndPoint,
remoteEndPoint: _remoteEndPoint))
{
var failureDetected = false;
var actualId = ConnectionId.None;
monitor.OnFailure += id =>
{
failureDetected = true;
actualId = id;
};
monitor.Start();
Thread.Sleep(TimeSpan.FromSeconds(value: 1));
Console.WriteLine("# heartbeats: {0}", monitor.NumHeartbeats);
const string reason = "Because the debugger is attached and no failures shall be reported when this is the case";
monitor.FailureDetected.Should().BeFalse(reason);
failureDetected.Should().BeFalse(reason);
actualId.Should().Be(ConnectionId.None);
failureStarted.Should().HaveValue();
monitor.NumHeartbeats.Should().BeGreaterOrEqualTo(expected: 25);
}
}
private void TestFailure(IDebugger debugger, bool enabledWithAttachedDebugger)
{
long actualNumHeartbeats = 0;
DateTime?failureStarted = null;
_heartbeat.Setup(x => x.Beat())
.Returns(() => Task.Factory.StartNew(() =>
{
// Let's simulate failure by blocking the task
if (++actualNumHeartbeats == 25)
{
failureStarted = DateTime.Now;
Thread.Sleep(millisecondsTimeout: 10000);
}
}));
HeartbeatMonitor monitor;
using (
monitor =
new HeartbeatMonitor(_heartbeat.Object,
debugger,
TimeSpan.FromSeconds(value: 0.01),
failureThreshold: 1,
enabledWithAttachedDebugger: enabledWithAttachedDebugger, useHeartbeatFailureDetection: true,
allowRemoteHeartbeatDisable: true, connectionId: _connectionId,
endPointName: "Test",
locEndPoint: _localEndPoint,
remoteEndPoint: _remoteEndPoint))
{
var failureDetected = false;
var actualId = ConnectionId.None;
monitor.OnFailure += id =>
{
failureDetected = true;
actualId = id;
};
monitor.Start();
Thread.Sleep(TimeSpan.FromSeconds(value: 1));
Console.WriteLine("# heartbeats: {0}", monitor.NumHeartbeats);
monitor.FailureDetected.Should().BeTrue();
failureDetected.Should().BeTrue();
actualId.Should().Be(_connectionId);
monitor.NumHeartbeats.Should().Be(expected: 24, because: "Because failure was initiated on the 25th heartbeat");
monitor.LastHeartbeat.Should().BeOnOrBefore(failureStarted.Value);
}
}
public void TestStart()
{
using (var monitor = new HeartbeatMonitor(_heartbeat.Object,
Debugger.Instance,
new HeartbeatSettings(),
_connectionId,
"Test",
_localEndPoint,
_remoteEndPoint))
{
monitor.IsStarted.Should().BeFalse();
monitor.Start();
monitor.IsStarted.Should().BeTrue();
}
}
public void TestTaskExceptionObservation()
{
var settings = new HeartbeatSettings
{
Interval = TimeSpan.FromMilliseconds(value: 10)
};
var exceptions = new List <Exception>();
TaskScheduler.UnobservedTaskException += (sender, args) => exceptions.Add(args.Exception);
using (var heartbeatFailure = new ManualResetEvent(initialState: false))
using (
var monitor = new HeartbeatMonitor(_heartbeat.Object,
_debugger.Object,
settings,
_connectionId,
"Test",
_localEndPoint,
_remoteEndPoint))
{
_heartbeat.Setup(x => x.Beat())
.Returns(() =>
{
var task = new Task(() =>
{
heartbeatFailure.WaitOne();
throw new ConnectionLostException();
});
task.Start();
return(task);
});
monitor.OnFailure += id => heartbeatFailure.Set();
monitor.Start();
heartbeatFailure.WaitOne(TimeSpan.FromMilliseconds(value: 500))
.Should().BeTrue("Because the task doesn't return before a failure was reported");
}
GC.Collect(generation: 2, mode: GCCollectionMode.Forced);
GC.WaitForPendingFinalizers();
exceptions.Should().Equal();
}
public void NormalConnectionTest()
{
// Arrange
var connection = new Mock<Client.IConnection>();
var transport = new Mock<IClientTransport>();
// Setting the values such that a timeout or timeout warning isn't issued
var keepAliveData = new KeepAliveData(
timeoutWarning: TimeSpan.FromSeconds(5),
timeout: TimeSpan.FromSeconds(10),
checkInterval: TimeSpan.FromSeconds(2)
);
using (var monitor = new HeartbeatMonitor(connection.Object, new object(), keepAliveData.CheckInterval))
{
connection.Setup(m => m.LastMessageAt).Returns(DateTime.UtcNow);
connection.Setup(m => m.KeepAliveData).Returns(keepAliveData);
connection.Setup(m => m.State).Returns(ConnectionState.Connected);
connection.Setup(m => m.Transport).Returns(transport.Object);
monitor.Start();
// Act - Setting timespan to be less than timeout and timeout warning
monitor.Beat(TimeSpan.FromSeconds(2));
// Assert
Assert.False(monitor.TimedOut);
Assert.False(monitor.HasBeenWarned);
}
}