public async Task PrimaryReplicaTest()
{
var clusterNotifierMock = new Mock <ClusterNotifier>();
var notifierSignal = new ManualResetEvent(false);
clusterNotifierMock.Setup(i => i.DistributeToCluster(It.IsAny <InflightMessage>(), It.IsAny <CancellationToken>(), It.IsAny <string>()))
.Callback((() => notifierSignal.Set())).Returns(Task.CompletedTask);
var replicaSet = new MockStatefulServiceReplicaSet <BullfrogNotificationBackendService.BullfrogNotificationBackendService>(
(sc, stateManagerReplica2) => new BullfrogNotificationBackendService.BullfrogNotificationBackendService(sc, stateManagerReplica2,
clusterNotifierMock.Object)
{
WaitTimeBetweenLoop = TimeSpan.FromMilliseconds(10)
}, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
await replicaSet.Primary.ServiceInstance.IngestMessage("blah", JsonMessage);
notifierSignal.WaitOne(TimeSpan.FromMilliseconds(30));
}
public async Task TestServiceState_InMemoryState_PromoteActiveSecondary()
{
var replicaSet = new MockStatefulServiceReplicaSet <MyStatefulService>(CreateStatefulService, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
const string stateName = "test";
var payload = new Payload(StatePayload);
//insert data
await replicaSet.Primary.ServiceInstance.InsertAsync(stateName, payload);
//promote one of the secondaries to primary
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
//get data
var payloads = (await replicaSet.Primary.ServiceInstance.GetPayloadsAsync()).ToList();
//data should match what was inserted against the primary
Assert.IsTrue(payloads.Count == 1);
Assert.IsTrue(payloads[0].Content == payload.Content);
//the primary should not have any in-memory state
var payloadsFromOldPrimary = await replicaSet[1].ServiceInstance.GetPayloadsAsync();
Assert.IsTrue(!payloadsFromOldPrimary.Any());
}
public async Task TestServiceState_InMemoryState_PromoteNewReplica()
{
var replicaSet = new MockStatefulServiceReplicaSet <MyStatefulService>(CreateStatefulService, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
var originalPrimary = replicaSet.Primary;
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
var originalSecondary = replicaSet.SecondaryReplicas.Single();
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
const string stateName = "test";
var payload = new Payload(StatePayload);
//insert data
await replicaSet.Primary.ServiceInstance.InsertAsync(stateName, payload);
//promote one of the secondaries to primary
await replicaSet.PromoteNewReplicaToPrimaryAsync(4);
//get data
var payloads = (await replicaSet.Primary.ServiceInstance.GetPayloadsAsync()).ToList();
//data should match what was inserted against the primary
Assert.IsTrue(payloads.Count == 1, "Unexpected payload count");
Assert.IsTrue(payloads[0].Content == payload.Content, "Unexpected payload content");
//the original primary should not have any in-memory state
var payloadsFromOldPrimary = await originalPrimary.ServiceInstance.GetPayloadsAsync();
Assert.IsFalse(payloadsFromOldPrimary.Any(), "Original primary payload should have been erased");
}
public async Task TestServiceStateChangesDuringPromote()
{
var replicaSet = new MockStatefulServiceReplicaSet <MyStatefulService>(CreateStatefulService, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
var originalPrimary = replicaSet.Primary;
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
var originalSecondary = replicaSet.SecondaryReplicas.Single();
//promote the secondary to primary
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
bool hasRun = originalPrimary.ServiceInstance.RunAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasRun, "RunAsync did not run on original primary");
hasRun = originalSecondary.ServiceInstance.RunAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasRun, "RunAsync did not run on new primary");
bool hasChanged = originalPrimary.ServiceInstance.ChangeRoleAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasChanged, "ChangeRole did not run on original primary");
hasRun = originalSecondary.ServiceInstance.ChangeRoleAsyncHasRun.WaitOne(500);
Assert.IsTrue(hasChanged, "ChangeRole did not run on new primary");
bool hasClearedCache = originalPrimary.ServiceInstance.CacheCleared.Wait(500);
Assert.IsTrue(hasChanged, "Cache was not cleared on original primary");
}
public async Task TestPrimaryReplicaShouldHaveOpenListenersAsync()
{
Func <StatefulServiceContext, IReliableStateManagerReplica2, StatefulServiceWithReplicaListener> serviceFactory = (StatefulServiceContext context, IReliableStateManagerReplica2 stateManager) => new StatefulServiceWithReplicaListener(context);
var replicaSet = new MockStatefulServiceReplicaSet <StatefulServiceWithReplicaListener>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
var openListeners = replicaSet.Primary.OpenListeners;
Assert.AreEqual(1, openListeners.Count());
}
public async Task TestPromoteActiveSecondaryToPrimaryAsync()
{
Func <StatefulServiceContext, IReliableStateManagerReplica2, StatefulServiceWithReplicaListener> serviceFactory = (StatefulServiceContext context, IReliableStateManagerReplica2 stateManager) => new StatefulServiceWithReplicaListener(context);
var replicaSet = new MockStatefulServiceReplicaSet <StatefulServiceWithReplicaListener>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
Assert.AreEqual(2, replicaSet.Primary.ReplicaId);
}
public async Task ReplicateSet_InitDataPassed()
{
var initData = Encoding.UTF8.GetBytes("blah");
var replicaSet = new MockStatefulServiceReplicaSet <MyStatefulService>(CreateStatefulService, CreateStateManagerReplica);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1, initializationData : initData);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2, initializationData : initData);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3, initializationData : initData);
CollectionAssert.AreEqual(Encoding.UTF8.GetBytes("blah"), replicaSet.Primary.ServiceInstance.Context.InitializationData);
foreach (var i in replicaSet.SecondaryReplicas)
{
CollectionAssert.AreEqual(Encoding.UTF8.GetBytes("blah"),
i.ServiceInstance.Context.InitializationData);
}
}
public async Task TestPromoteActiveSecondaryToPrimaryWithRunAsyncOverrideAsync()
{
Func <StatefulServiceContext, IReliableStateManagerReplica2, StatefulServiceWithRunAsyncOverride> serviceFactory = (StatefulServiceContext context, IReliableStateManagerReplica2 stateManager) => new StatefulServiceWithRunAsyncOverride(context);
var replicaSet = new MockStatefulServiceReplicaSet <StatefulServiceWithRunAsyncOverride>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
var originalPrimaryReplica = replicaSet.Primary;
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
var originalSecondaryReplica = replicaSet.SecondaryReplicas.Single();
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
bool completed = originalPrimaryReplica.ServiceInstance.RunAsyncCompleted.WaitOne(500, false);
Assert.IsTrue(completed);
Assert.AreEqual(2, replicaSet.Primary.ReplicaId);
originalPrimaryReplica.RunCancellation.Cancel();
originalSecondaryReplica.RunCancellation.Cancel();
}
public void TestDeadLock9()
{
Func <StatefulServiceContext, IReliableStateManagerReplica2, StatefulServiceWithReplicaListener> serviceFactory = (context, stateManagerReplica) =>
{
var partition = new MockStatefulServicePartition()
{
PartitionInfo = MockQueryPartitionFactory.CreateSingletonPartitonInfo(Guid.NewGuid())
};
context = MockStatefulServiceContextFactory.Create(
context.CodePackageActivationContext,
context.ServiceTypeName,
context.ServiceName,
partition.PartitionInfo.Id,
context.ReplicaId);
var service = new StatefulServiceWithReplicaListener(context);
service.SetPartition(partition);
return(service);
};
//shared instance, called from multiple threads
MockStatefulServiceReplicaSet <StatefulServiceWithReplicaListener> replicaSet;
Parallel.For(1, 100, async(i) =>
{
replicaSet = new MockStatefulServiceReplicaSet <StatefulServiceWithReplicaListener>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 4);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 5);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 6);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 7);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 8);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 9);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 10);
});
}
public async Task TestServiceState_InMemoryState_PromoteActiveSecondary()
{
var stateManager = new MockReliableStateManager();
//service factory to instruct how to create the service instance
Func <StatefulServiceContext, IReliableStateManagerReplica2, UserStatefulService.UserStatefulService> serviceFactory = (StatefulServiceContext context, IReliableStateManagerReplica2 stateManager1) => new UserStatefulService.UserStatefulService(context);
var replicaSet = new MockStatefulServiceReplicaSet <UserStatefulService.UserStatefulService>(serviceFactory);
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2);
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3);
const string stateName = "test";
// var payload = new Payload(StatePayload);
//insert data
CancellationToken token = new CancellationToken(false);
await replicaSet.Primary.ServiceInstance.InvokeRunAsync(token);
//promote one of the secondaries to primary
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(2);
//get data
// var payloads = (await replicaSet.Primary.ServiceInstance.Context.()).ToList();
//data should match what was inserted against the primary
// Assert.IsTrue(payloads.Count == 1);
// Assert.IsTrue(payloads[0].Content == payload.Content);
//verify the data was saved against the reliable dictionary
//var dictionary = await StateManager.GetOrAddAsync<IReliableDictionary<string, Payload>>(MyStatefulService.StateManagerDictionaryKey);
//using (var tx = StateManager.CreateTransaction())
//{
// var payload = await dictionary.TryGetValue(stateName);
// Assert.IsTrue(payload.HasValue);
// Assert.IsTrue(payload.Value.Content == payload.Content);
//}
}
public async Task PromoteActivateSecondaryToPrimary(string eventName, int messageCount)
{
for (var x = 1; x <= messageCount; x++)
{
_mockActorProxyFactory.RegisterActor(
CreateMockEventHandlerActor(new ActorId(string.Format(eventName + "-{0}", x))));
}
var count = 0;
_mockMessageProvider.Setup(s => s.ReceiveAsync(
It.IsAny <int>(),
It.IsAny <TimeSpan>())).ReturnsAsync(() =>
{
if (count >= messageCount)
{
return(new List <Message>());
}
count++;
return(CreateMessage(eventName));
});
var mockServiceBusManager = new Mock <IServiceBusManager>();
mockServiceBusManager.Setup(s => s.CreateAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>()));
mockServiceBusManager.Setup(s => s.CreateMessageReceiver(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <bool>())).Returns(_mockMessageProvider.Object);
mockServiceBusManager.Setup(s => s.GetLockToken(It.IsAny <Message>())).Returns(Guid.NewGuid().ToString);
EventReaderService.EventReaderService Factory(StatefulServiceContext context, IReliableStateManagerReplica2 stateManager) =>
new EventReaderService.EventReaderService(
context,
stateManager,
_mockedBigBrother,
mockServiceBusManager.Object,
_mockActorProxyFactory,
_config);
var replicaSet = new MockStatefulServiceReplicaSet <EventReaderService.EventReaderService>(Factory, (context, dictionary) => new MockReliableStateManager(dictionary));
//add a new Primary replica
await replicaSet.AddReplicaAsync(ReplicaRole.Primary, 1, initializationData : Encoding.UTF8.GetBytes("test.type"));
//add a new ActiveSecondary replica
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 2, initializationData : Encoding.UTF8.GetBytes("test.type"));
//add a second ActiveSecondary replica
await replicaSet.AddReplicaAsync(ReplicaRole.ActiveSecondary, 3, initializationData : Encoding.UTF8.GetBytes("test.type"));
void CheckInFlightMessagesOnPrimary()
{
var innerTask = Task.Run(async() =>
{
while (replicaSet.Primary.ServiceInstance.InFlightMessageCount != messageCount)
{
await Task.Delay(100);
}
});
innerTask.Wait(TimeSpan.FromSeconds(2)).Should().BeTrue();
}
CheckInFlightMessagesOnPrimary();
var oldPrimaryReplicaId = replicaSet.Primary.ReplicaId;
await replicaSet.PromoteActiveSecondaryToPrimaryAsync(replicaSet.FirstActiveSecondary.ReplicaId);
replicaSet.Primary.ReplicaId.Should().NotBe(oldPrimaryReplicaId);
CheckInFlightMessagesOnPrimary();
}
