public async Task TestServiceStateChangesDuringPromote()
{
var replicaSet = new MockStatefulServiceReplicaSet <MyStatefulService>(CreateStatefulService, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
var originalPrimary = replicaSet.Primary;
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
var originalSecondary = replicaSet.SecondaryReplicas.Single();
//promote the secondary to primary
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
bool hasRun = originalPrimary.ServiceInstance.RunAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasRun, "RunAsync did not run on original primary");
hasRun = originalSecondary.ServiceInstance.RunAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasRun, "RunAsync did not run on new primary");
bool hasChanged = originalPrimary.ServiceInstance.ChangeRoleAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasChanged, "ChangeRole did not run on original primary");
hasRun = originalSecondary.ServiceInstance.ChangeRoleAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasChanged, "ChangeRole did not run on new primary");
bool hasClearedCache = originalPrimary.ServiceInstance.CacheCleared.Wait(500);
Assert.IsTrue(hasChanged, "Cache was not cleared on original primary");
}
public async Task TestServiceState_InMemoryState_PromoteActiveSecondary()
{
var replicaSet = new MockStatefulServiceReplicaSet <MyStatefulService>(CreateStatefulService, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
const string stateName = "test";
var payload = new Payload(StatePayload);
//insert data
await replicaSet.Primary.ServiceInstance.InsertAsync(stateName, payload);
//promote one of the secondaries to primary
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
//get data
var payloads = (await replicaSet.Primary.ServiceInstance.GetPayloadsAsync()).ToList();
//data should match what was inserted against the primary
Assert.IsTrue(payloads.Count == 1);
Assert.IsTrue(payloads[0].Content == payload.Content);
//the primary should not have any in-memory state
var payloadsFromOldPrimary = await replicaSet[1].ServiceInstance.GetPayloadsAsync();
Assert.IsTrue(!payloadsFromOldPrimary.Any());
}
public async Task TestPromoteActiveSecondaryToPrimaryAsync()
{
Func <StatefulServiceContext, IReliableStateManagerReplica2, StatefulServiceWithReplicaListener> serviceFactory = (StatefulServiceContext context, IReliableStateManagerReplica2 stateManager) => new StatefulServiceWithReplicaListener(context);
var replicaSet = new MockStatefulServiceReplicaSet <StatefulServiceWithReplicaListener>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
Assert.AreEqual(2, replicaSet.Primary.ReplicaId);
}
public async Task TestPromoteActiveSecondaryToPrimaryWithRunAsyncOverrideAsync()
{
Func <StatefulServiceContext, IReliableStateManagerReplica2, StatefulServiceWithRunAsyncOverride> serviceFactory = (StatefulServiceContext context, IReliableStateManagerReplica2 stateManager) => new StatefulServiceWithRunAsyncOverride(context);
var replicaSet = new MockStatefulServiceReplicaSet <StatefulServiceWithRunAsyncOverride>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
var originalPrimaryReplica = replicaSet.Primary;
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
var originalSecondaryReplica = replicaSet.SecondaryReplicas.Single();
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
bool completed = originalPrimaryReplica.ServiceInstance.RunAsyncCompleted.WaitOne(500, false);
Assert.IsTrue(completed);
Assert.AreEqual(2, replicaSet.Primary.ReplicaId);
originalPrimaryReplica.RunCancellation.Cancel();
originalSecondaryReplica.RunCancellation.Cancel();
}
public async Task TestServiceState_InMemoryState_PromoteActiveSecondary_WithLongRunAsync()
{
var replicaSet = new MockStatefulServiceReplicaSet <MyLongRunningStatefulService>(CreateLongRunningStatefulService, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
var originalPrimary = replicaSet.Primary;
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
var originalSecondary = replicaSet.SecondaryReplicas.Single();
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
const string stateName = "test";
var payload = new Payload(StatePayload);
//insert data
await replicaSet.Primary.ServiceInstance.InsertAsync(stateName, payload);
//promote one of the secondaries to primary
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
//wait
bool hasRun = originalPrimary.ServiceInstance.RunAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasRun, "RunAsync did not run on original primary");
hasRun = originalSecondary.ServiceInstance.RunAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasRun, "RunAsync did not run on new primary");
//get data
var payloads = (await replicaSet.Primary.ServiceInstance.GetPayloadsAsync()).ToList();
//data should match what was inserted against the primary
Assert.IsTrue(payloads.Count == 1, "Unexpected payload count");
Assert.IsTrue(payloads[0].Content == payload.Content, "Unexpected payload content");
//the original primary should not have any in-memory state
var payloadsFromOldPrimary = await originalPrimary.ServiceInstance.GetPayloadsAsync();
Assert.IsFalse(payloadsFromOldPrimary.Any(), "Original primary payload should have been erased");
}
public async Task TestServiceState_InMemoryState_PromoteActiveSecondary()
{
var stateManager = new MockReliableStateManager();
//service factory to instruct how to create the service instance
Func <StatefulServiceContext, IReliableStateManagerReplica2, UserStatefulService.UserStatefulService> serviceFactory = (StatefulServiceContext context, IReliableStateManagerReplica2 stateManager1) => new UserStatefulService.UserStatefulService(context);
var replicaSet = new MockStatefulServiceReplicaSet <UserStatefulService.UserStatefulService>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
const string stateName = "test";
// var payload = new Payload(StatePayload);
//insert data
CancellationToken token = new CancellationToken(false);
await replicaSet.Primary.ServiceInstance.InvokeRunAsync(token);
//promote one of the secondaries to primary
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
//get data
// var payloads = (await replicaSet.Primary.ServiceInstance.Context.()).ToList();
//data should match what was inserted against the primary
// Assert.IsTrue(payloads.Count == 1);
// Assert.IsTrue(payloads[0].Content == payload.Content);
//verify the data was saved against the reliable dictionary
//var dictionary = await StateManager.GetOrAddAsync<IReliableDictionary<string, Payload>>(MyStatefulService.StateManagerDictionaryKey);
//using (var tx = StateManager.CreateTransaction())
//{
// var payload = await dictionary.TryGetValue(stateName);
// Assert.IsTrue(payload.HasValue);
// Assert.IsTrue(payload.Value.Content == payload.Content);
//}
}
public async Task PromoteActivateSecondaryToPrimary(string eventName, int messageCount)
{
for (var x = 1; x <= messageCount; x++)
{
_mockActorProxyFactory.RegisterActor(
CreateMockEventHandlerActor(new ActorId(string.Format(eventName + "-{0}", x))));
}
var count = 0;
_mockMessageProvider.Setup(s => s.ReceiveAsync(
It.IsAny <int>(),
It.IsAny <TimeSpan>())).ReturnsAsync(() =>
{
if (count >= messageCount)
{
return(new List <Message>());
}
count++;
return(CreateMessage(eventName));
});
var mockServiceBusManager = new Mock <IServiceBusManager>();
mockServiceBusManager.Setup(s => s.CreateAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>()));
mockServiceBusManager.Setup(s => s.CreateMessageReceiver(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <bool>())).Returns(_mockMessageProvider.Object);
mockServiceBusManager.Setup(s => s.GetLockToken(It.IsAny <Message>())).Returns(Guid.NewGuid().ToString);
EventReaderService.EventReaderService Factory(StatefulServiceContext context, IReliableStateManagerReplica2 stateManager) =>
new EventReaderService.EventReaderService(
context,
stateManager,
_mockedBigBrother,
mockServiceBusManager.Object,
_mockActorProxyFactory,
_config);
var replicaSet = new MockStatefulServiceReplicaSet <EventReaderService.EventReaderService>(Factory, (context, dictionary) => new MockReliableStateManager(dictionary));
//add a new Primary replica
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1, initializationData : Encoding.UTF8.GetBytes("test.type"));
//add a new ActiveSecondary replica
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2, initializationData : Encoding.UTF8.GetBytes("test.type"));
//add a second ActiveSecondary replica
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3, initializationData : Encoding.UTF8.GetBytes("test.type"));
void CheckInFlightMessagesOnPrimary()
{
var innerTask = Task.Run(async() =>
{
while (replicaSet.Primary.ServiceInstance.InFlightMessageCount != messageCount)
{
await Task.Delay(100);
}
});
innerTask.Wait(TimeSpan.FromSeconds(2)).Should().BeTrue();
}
CheckInFlightMessagesOnPrimary();
var oldPrimaryReplicaId = replicaSet.Primary.ReplicaId;
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(replicaSet.FirstActiveSecondary.ReplicaId);
replicaSet.Primary.ReplicaId.Should().NotBe(oldPrimaryReplicaId);
CheckInFlightMessagesOnPrimary();
}