public async Task should_block_indefinitely_if_a_leadership_lease_can_not_be_obtained()
{
var service = A.Fake <ITestService>();
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(false);
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStopping(new CancellationTokenSource(1000))
.Lease(lcb =>
{
lcb.RenewLeaseEvery(TimeSpan.FromMilliseconds(50))
.AquireLeaseEvery(TimeSpan.FromMilliseconds(100))
.WithLeaseManager(lc => manager);
})
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.Build();
var runner = new Runner <ITestService>(service, config);
await runner.Start();
A.CallTo(() => service.Start(A <CancellationToken> .Ignored)).MustHaveHappened(0, Times.Exactly);
A.CallTo(manager).MustHaveHappened(1, Times.OrMore);
}
public void guard_against_invalid_health_check_times(int seconds)
{
var healthCheckEvery = TimeSpan.FromSeconds(seconds);
var builder = new LeaderConfigurationBuilder <object>();
Assert.Throws <ArgumentOutOfRangeException>(() => builder.AttemptToBeTheLeaderEvery(healthCheckEvery));
}
public void guard_against_invalid_lease_renewal_times(int seconds)
{
var leaseUpdate = TimeSpan.FromSeconds(seconds);
var builder = new LeaderConfigurationBuilder <object>();
Assert.Throws <ArgumentOutOfRangeException>(() => builder.UpdateLeaseEvery(leaseUpdate));
}
public void use_the_in_memory_lease_manager_when_an_alternative_manager_isnt_configured()
{
var builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => { Console.WriteLine("Started"); return(Task.CompletedTask); });
var built = builder.Build();
Assert.IsType <InMemoryLeaseManager>(built.LeaseManager);
}
public void use_the_lease_manager_that_is_provided()
{
var manager = A.Fake <ILeaseManager>();
var builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => Task.CompletedTask);
builder.Lease(managerBuilder => managerBuilder.WithLeaseManager((c) => manager));
Assert.Same(manager, builder.Build().LeaseManager);
}
public void use_the_lease_renewal_time_that_is_provided()
{
var leaseUpdate = TimeSpan.FromDays(1);
var builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => Task.CompletedTask);
builder.UpdateLeaseEvery(leaseUpdate);
Assert.Equal(leaseUpdate, builder.Build().LeaseUpdateEvery);
}
public void use_the_nodeid_that_is_provided()
{
const string nodeid = "testvalue";
var builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => Task.CompletedTask);
builder.SetNodeId(nodeid);
Assert.Equal(nodeid, builder.Build().NodeId);
}
public void use_the_health_check_time_that_is_provided()
{
var healthCheckEvery = TimeSpan.FromDays(1);
var builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => Task.CompletedTask);
builder.AttemptToBeTheLeaderEvery(healthCheckEvery);
Assert.Equal(healthCheckEvery, builder.Build().LeaderCheckEvery);
}
public async Task should_invoke_the_heartbeat_at_set_intervals(bool isLeader)
{
var service = BuildBlockingTestService();
var manager = A.Fake <ILeaseManager>();
const int milliSecondsBeforeServiceStops = 2000;
const int milliSecondsBetweenHeartBeats = 200;
var invocationTimes = new List <DateTime>();
A.CallTo(() => manager.AcquireLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(isLeader));
A.CallTo(() => manager.RenewLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(isLeader));
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStopping(new CancellationTokenSource(milliSecondsBeforeServiceStops))
.Lease(lcb => lcb.WithLeaseManager(manager))
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.WithHeartBeat(
TimeSpan.FromMilliseconds(milliSecondsBetweenHeartBeats),
(b, token) =>
{
invocationTimes.Add(DateTime.UtcNow);
return(Task.CompletedTask);
})
.Build();
var runner = new Runner <ITestService>(service, config);
await runner.Start();
for (var i = 0; i < invocationTimes.Count; i++)
{
if (i == 0)
{
this.output.WriteLine($"Invocation time: {invocationTimes[i]:HH:mm:ss.fff}");
continue;
}
var invocationTime = invocationTimes[i];
var millisecondsDifferenceBetweenInvocations = invocationTime.Subtract(invocationTimes[i - 1]).TotalMilliseconds;
this.output.WriteLine($"Invocation time: {invocationTime:HH:mm:ss.fff}. Difference between previous invocation (ms): {millisecondsDifferenceBetweenInvocations}.");
Assert.InRange(
millisecondsDifferenceBetweenInvocations,
milliSecondsBetweenHeartBeats * 0.9,
milliSecondsBetweenHeartBeats * 1.1);
}
var expectedNumberOfInvocations = milliSecondsBeforeServiceStops / milliSecondsBetweenHeartBeats;
this.output.WriteLine($"Actual no. of invocations: {invocationTimes.Count}. Expected no. of invocations: {expectedNumberOfInvocations}.");
Assert.Equal(invocationTimes.Count, expectedNumberOfInvocations);
}
public void set_a_unique_nodeid_if_one_isnt_provided()
{
var builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => Task.CompletedTask);
var firstId = builder.Build().NodeId;
builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => Task.CompletedTask);
var secondId = builder.Build().NodeId;
Assert.NotStrictEqual(firstId, secondId);
}
private static void BuildSubject(string nodeid, ILeaseManager manager, Action <bool> whenLeaderElected, out TestService servicewithStopSupport, out Runner <TestService> runner)
{
var config = new LeaderConfigurationBuilder <TestService>()
.SetNodeId(nodeid)
.AttemptToBeTheLeaderEvery(TimeSpan.FromMilliseconds(100))
.UpdateLeaseEvery(TimeSpan.FromMilliseconds(50))
.WhenStarted(async(s, token) => await s.Start(token))
.WhenLeaderIsElected(whenLeaderElected)
.WithLeaseManager(manager);
servicewithStopSupport = new TestService();
runner = new Runner <TestService>(servicewithStopSupport, config.Build());
}
public void use_the_hearbeat_that_was_provided()
{
var heartBeatInterval = TimeSpan.Zero;
var onHeartBeat = new Func <bool, CancellationToken, Task>((isActive, token) => Task.CompletedTask);
var builder = new LeaderConfigurationBuilder <object>();
builder.WhenStarted((o, token) => Task.CompletedTask);
builder.WithHeartBeat(heartBeatInterval, onHeartBeat);
var config = builder.Build();
Assert.Equal(heartBeatInterval, config.HeartBeatInterval);
Assert.Equal(onHeartBeat, config.OnHeartBeat);
}
public async Task stop_running_when_the_service_is_stopping()
{
var service = new TestService();
var stopping = new CancellationTokenSource();
var config = new LeaderConfigurationBuilder <TestService>()
.WhenStopping(stopping)
.WhenStarted(async(testService, token) => await testService.Start(token))
.Build();
var runner = new Runner <TestService>(service, config);
var startTask = runner.Start();
stopping.Cancel();
await startTask;
Assert.True(startTask.IsCompleted);
}
public void provide_the_lease_calculator_to_the_lease_manager()
{
var fakeCalculator = A.Fake <ILeaseLengthCalculator>();
var lcb = new LeaderConfigurationBuilder <object>();
ILeaseLengthCalculator calculatorProvided = null;
lcb.WhenStarted((o, token) => throw new Exception())
.Lease(cb =>
{
cb.WithLeaseManager(lc =>
{
calculatorProvided = lc.LeaseLengthCalculator;
return(new InMemoryLeaseManager(NodeId));
}).LeaseLength(() => fakeCalculator);
});
lcb.Build();
Assert.Same(fakeCalculator, calculatorProvided);
}
public async Task should_attempt_to_obtain_a_lease_again_if_the_lease_could_not_be_renewed()
{
var service = BuildBlockingTestService();
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
A.CallTo(() => manager.RenewLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(false));
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStopping(new CancellationTokenSource(1000))
.Lease(lcb => { lcb.RenewLeaseEvery(TimeSpan.FromMilliseconds(50)).WithLeaseManager(manager); })
.WhenStarted(async(svc, token) => { await svc.Start(token); })
.Build();
var runner = new Runner <ITestService>(service, config);
await runner.Start();
A.CallTo(() => manager.AcquireLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).MustHaveHappened(2, Times.OrMore);
}
public async Task should_bubble_exceptions_in_the_service()
{
var exception = new Exception();
var service = BuildBadTestService(exception);
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.Lease(lcb => lcb.WithLeaseManager(manager))
.Build();
var thrownException = await Assert.ThrowsAsync <AggregateException>(async() => await new Runner <ITestService>(service, config).Start());
Assert.Contains(exception, thrownException.InnerExceptions);
}
public async Task should_set_the_cancellation_token_when_unhandled_exceptions_occur_in_the_service()
{
var exception = new Exception();
var service = BuildBadTestService(exception);
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
var serviceStopping = new CancellationTokenSource();
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStopping(serviceStopping)
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.Lease(lcb => lcb.WithLeaseManager(manager))
.Build();
await Assert.ThrowsAnyAsync <AggregateException>(async() => await new Runner <ITestService>(service, config).Start());
Assert.True(serviceStopping.IsCancellationRequested);
}
public async Task should_bubble_exceptions_if_the_service_and_leadershipmanager_throw_exceptions()
{
var serviceException = new Exception("Service stopped working");
var leadershipManagerException = new Exception("Leadership manager stopped working");
var service = BuildBadTestService(serviceException);
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <string> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
A.CallTo(() => manager.RenewLease(A <string> .Ignored, A <CancellationToken> .Ignored)).ThrowsAsync(leadershipManagerException);
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.WithLeaseManager(manager)
.Build();
var thrownException = await Assert.ThrowsAsync <AggregateException>(async() => await new Runner <ITestService>(service, config).Start());
Assert.Contains(serviceException, thrownException.InnerExceptions);
Assert.Contains(leadershipManagerException, thrownException.InnerExceptions);
}
public async Task should_start_if_a_leadership_lease_is_obtained()
{
var service = BuildBlockingTestService();
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
A.CallTo(() => manager.RenewLease(A <LeaseOptions> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStopping(new CancellationTokenSource(1000))
.Lease(lcb => { lcb.RenewLeaseEvery(TimeSpan.FromMilliseconds(50)).WithLeaseManager(manager); })
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.Build();
var runner = new Runner <ITestService>(service, config);
await runner.Start();
A.CallTo(() => service.Start(A <CancellationToken> .Ignored)).MustHaveHappened(Repeated.Exactly.Once);
}
public async Task should_attempt_to_renew_a_lease_once_it_has_obtained_one()
{
var service = BuildBlockingTestService();
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <string> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
A.CallTo(() => manager.RenewLease(A <string> .Ignored, A <CancellationToken> .Ignored)).Returns(Task.FromResult(true));
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStopping(new CancellationTokenSource(1000))
.UpdateLeaseEvery(TimeSpan.FromMilliseconds(50))
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.WithLeaseManager(manager)
.Build();
var runner = new Runner <ITestService>(service, config);
await runner.Start();
A.CallTo(() => manager.RenewLease(A <string> .Ignored, A <CancellationToken> .Ignored)).MustHaveHappened(Repeated.AtLeast.Twice);
}
public async Task should_block_indefinitely_if_a_leadership_lease_can_not_be_obtained()
{
var service = A.Fake <ITestService>();
var manager = A.Fake <ILeaseManager>();
A.CallTo(() => manager.AcquireLease(A <string> .Ignored, A <CancellationToken> .Ignored)).Returns(false);
var config = new LeaderConfigurationBuilder <ITestService>()
.WhenStopping(new CancellationTokenSource(1000))
.AttemptToBeTheLeaderEvery(TimeSpan.FromMilliseconds(100))
.WhenStarted(async(svc, token) =>
{
await svc.Start(token);
})
.WithLeaseManager(manager)
.Build();
var runner = new Runner <ITestService>(service, config);
await runner.Start();
A.CallTo(() => service.Start(A <CancellationToken> .Ignored)).MustHaveHappened(Repeated.Never);
A.CallTo(manager).MustHaveHappened(Repeated.AtLeast.Once);
}
public void prevent_a_leader_configuration_which_doesnt_handle_starting_up()
{
var builder = new LeaderConfigurationBuilder <object>();
Assert.Throws <HostConfigurationException>(() => builder.Build());
}