public void AccrualFailureDetector_must_return_phi_based_on_guess_when_only_one_heartbeat()
{
var timeInterval = new List <long>()
{
0, 1000, 1000, 1000, 1000
};
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(8.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.Zero,
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeInterval));
fd.HeartBeat();
ShouldBe(fd.CurrentPhi, 0.3D, 0.2D);
ShouldBe(fd.CurrentPhi, 4.5D, 0.3D);
Assert.True(fd.CurrentPhi > 15.0D);
}
public void AccrualFailureDetector_must_return_phi_value_using_first_interval_after_second_heartbeat()
{
var timeInterval = new List <long>()
{
0, 100, 100, 100, 100
};
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(8.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.Zero,
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeInterval));
fd.HeartBeat();
Assert.True(fd.CurrentPhi > 0.0d);
fd.HeartBeat();
Assert.True(fd.CurrentPhi > 0.0d);
}
public void AccrualFailureDetector_must_fail_after_configured_acceptable_missing_heartbeats()
{
var timeIntervals = new List <long>()
{
0, 1000, 1000, 1000, 1000, 1000, 500, 500, 5000
};
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(8.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3),
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeIntervals));
fd.HeartBeat();
fd.HeartBeat();
fd.HeartBeat();
fd.HeartBeat();
fd.HeartBeat();
fd.HeartBeat();
Assert.True(fd.IsAvailable);
fd.HeartBeat();
Assert.False(fd.IsAvailable);
}
public void AccrualFailureDetector_must_use_maxSampleSize_heartbeats()
{
var timeIntervals = new List <long>()
{
0, 100, 100, 100, 100, 600, 500, 500, 500, 500, 500
};
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(8.0d, 3, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3),
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeIntervals));
// 100 ms interval
fd.HeartBeat(); //0
fd.HeartBeat(); //100
fd.HeartBeat(); //200
fd.HeartBeat(); //300
var phi1 = fd.CurrentPhi; //400
// 500 ms interval, should become same phi when 100 ms intervals have been dropped
fd.HeartBeat(); //1000
fd.HeartBeat(); //1500
fd.HeartBeat(); //2000
fd.HeartBeat(); //2500
var phi2 = fd.CurrentPhi; //3000
ShouldBe(phi1, phi2);
}
public void AccrualFailureDetector_must_accept_some_configured_missing_heartbeats()
{
var timeIntervals = new List <long>()
{
0L, 1000L, 1000L, 1000L, 4000L, 1000L, 1000L
};
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(8.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3), //3 seconds acceptableLostDuration
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeIntervals));
fd.HeartBeat();
fd.HeartBeat();
fd.HeartBeat();
fd.HeartBeat();
Assert.True(fd.IsAvailable);
fd.HeartBeat();
Assert.True(fd.IsAvailable);
}
public void AccrualFailureDetector_must_mark_node_as_available_if_it_starts_heartbeats_again_after_being_marked_dead_due_to_detection_of_failure
()
{
//1000 regular intervals, 5 minute pause, then a short pause again that should trigger unreachable again
var regularIntervals = new List <long>()
{
0L
}.Concat(Enumerable.Repeat(1000L, 999)).ToList();
var timeIntervals = regularIntervals.Concat(new[] { (5 * 60 * 1000L), 100L, 900L, 100L, 7000L, 100L, 900L, 100L, 900L }).ToList();
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(8.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3),
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeIntervals));
for (var i = 0; i < 1000; i++)
{
fd.HeartBeat();
}
Assert.False(fd.IsAvailable); //after the long pause
fd.HeartBeat();
Assert.True(fd.IsAvailable);
fd.HeartBeat();
Assert.False(fd.IsAvailable); //after the 7 seconds pause
fd.HeartBeat();
Assert.True(fd.IsAvailable);
fd.HeartBeat();
Assert.True(fd.IsAvailable);
}
public void AccrualFailureDetector_must_mark_node_as_dead_if_heartbeats_are_missed()
{
var timeInterval = new List <long>()
{
0, 1000, 100, 100, 7000
};
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(3.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.Zero,
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeInterval));
fd.HeartBeat(); //0
fd.HeartBeat(); //1000
fd.HeartBeat(); //1100
Assert.True(fd.IsAvailable); //1200
Assert.False(fd.IsAvailable); //8200
}
public void AccrualFailureDetector_must_mark_node_as_monitored_after_a_series_of_successful_heartbeats()
{
var timeInterval = new List <long>()
{
0, 1000, 100, 100
};
var fd = FailureDetectorSpecHelpers.CreateFailureDetector(8.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.Zero,
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeInterval));
Assert.False(fd.IsMonitoring);
fd.HeartBeat();
fd.HeartBeat();
fd.HeartBeat();
Assert.True(fd.IsMonitoring);
Assert.True(fd.IsAvailable);
}
public void FailureDetectorRegistry_must_fail_after_configured_acceptable_missing_heartbeats()
{
var timeIntervals = new List <long>()
{
0, 1000, 1000, 1000, 1000, 1000, 500, 500, 5000
};
var fd =
FailureDetectorSpecHelpers.CreateFailureDetectorRegistry(8.0d, 1000, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3), //changed to 3 seconds
TimeSpan.FromSeconds(1), FailureDetectorSpecHelpers.FakeTimeGenerator(timeIntervals));
fd.Heartbeat("resource1");
fd.Heartbeat("resource1");
fd.Heartbeat("resource1");
fd.Heartbeat("resource1");
fd.Heartbeat("resource1");
fd.Heartbeat("resource1");
Assert.True(fd.IsAvailable("resource1"));
fd.Heartbeat("resource1");
Assert.False(fd.IsAvailable("resource1"));
}