public void Cluster_CRDT_should_work_in_single_node_cluster()
{
Join(_first, _first);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(5.0), () =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(1));
});
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref));
_replicator.Tell(Dsl.Subscribe(KeyX, changedProbe.Ref));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new NotFound(KeyA, null));
var c3 = GCounter.Empty.Increment(_cluster, 3);
var update = Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 3));
_replicator.Tell(update);
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c3));
changedProbe.ExpectMsg(new Changed(KeyA, c3));
var changedProbe2 = CreateTestProbe();
_replicator.Tell(new Subscribe(KeyA, changedProbe2.Ref));
changedProbe2.ExpectMsg(new Changed(KeyA, c3));
var c4 = c3.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeTwo, x => x.Increment(_cluster)));
ExpectMsg(new UpdateTimeout(KeyA, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c4));
changedProbe.ExpectMsg(new Changed(KeyA, c4));
var c5 = c4.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeMajority, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readMajority));
ExpectMsg(new GetSuccess(KeyA, null, c5));
changedProbe.ExpectMsg(new Changed(KeyA, c5));
var c6 = c5.Increment(_cluster);
_replicator.Tell(Dsl.Update(KeyA, _writeAll, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readAll));
ExpectMsg(new GetSuccess(KeyA, null, c6));
changedProbe.ExpectMsg(new Changed(KeyA, c6));
var c9 = GCounter.Empty.Increment(_cluster, 9);
_replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 9)));
ExpectMsg(new UpdateSuccess(KeyX, null));
changedProbe.ExpectMsg(new Changed(KeyX, c9));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DeleteSuccess(KeyX));
changedProbe.ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, _readAll));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Update(KeyX, WriteLocal.Instance, x => x.Increment(_cluster)));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.GetKeyIds);
ExpectMsg(new GetKeysIdsResult(ImmutableHashSet <string> .Empty.Add("A")));
}, _first);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_be_replicated_after_successful_update()
{
var changedProbe = CreateTestProbe();
RunOn(() =>
{
_replicator.Tell(Dsl.Subscribe(KeyC, changedProbe.Ref));
}, _first, _second);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
_replicator.Tell(Dsl.Update(KeyY, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyY, null));
_replicator.Tell(Dsl.Update(KeyZ, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyZ, null));
}, _first);
EnterBarrier("update-c30");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c30 = ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyC)).Get(KeyC);
c30.Value.ShouldBe(30);
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
// replicate with gossip after WriteLocal
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
_replicator.Tell(Dsl.Delete(KeyY, WriteLocal.Instance, 777));
ExpectMsg(new DeleteSuccess(KeyY, 777));
_replicator.Tell(Dsl.Get(KeyZ, _readMajority));
ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyZ)).Get(KeyZ).Value.ShouldBe(30);
}, _second);
EnterBarrier("update-c31");
RunOn(() =>
{
// KeyC and deleted KeyY should be replicated via gossip to the other node
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(31);
_replicator.Tell(Dsl.Get(KeyY, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyY));
});
});
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
}, _first);
EnterBarrier("verified-c31");
// and also for concurrent updates
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c31 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c31.Value.ShouldBe(31);
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(33);
});
});
}, _first, _second);
EnterBarrierAfterTestStep();
}
public void Replicator_in_chaotic_cluster_should_replicate_data_in_initial_phase()
{
Join(First, First);
Join(Second, First);
Join(Third, First);
Join(Fourth, First);
Join(Fifth, First);
Within(TimeSpan.FromSeconds(10), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(5));
});
});
RunOn(() =>
{
for (int i = 0; i < 5; i++)
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
_replicator.Tell(Dsl.Update(KeyB, PNCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 1)));
}
ReceiveN(15).Select(x => x.GetType()).ToImmutableHashSet().ShouldBe(new[] { typeof(UpdateSuccess) });
}, First);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 20)));
_replicator.Tell(Dsl.Update(KeyB, PNCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 20)));
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 20)));
ReceiveN(3).ToImmutableHashSet().ShouldBe(new[]
{
new UpdateSuccess(KeyA, null),
new UpdateSuccess(KeyB, null),
new UpdateSuccess(KeyC, null)
});
_replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, WriteLocal.Instance, x => x.Add("e1").Add("e2")));
ExpectMsg(new UpdateSuccess(KeyE, null));
_replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, WriteLocal.Instance, x => x
.Add(_cluster, "e1")
.Add(_cluster, "e2")));
ExpectMsg(new UpdateSuccess(KeyF, null));
}, Second);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 40)));
ExpectMsg(new UpdateSuccess(KeyD, null));
_replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, WriteLocal.Instance, x => x.Add("e2").Add("e3")));
ExpectMsg(new UpdateSuccess(KeyE, null));
_replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, WriteLocal.Instance, x => x
.Add(_cluster, "e2")
.Add(_cluster, "e3")));
ExpectMsg(new UpdateSuccess(KeyF, null));
}, Fourth);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 50)));
ExpectMsg(new UpdateSuccess(KeyX, null));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DeleteSuccess(KeyX));
}, Fifth);
EnterBarrier("initial-updates-done");
AssertValue(KeyA, 25);
AssertValue(KeyB, 15);
AssertValue(KeyC, 25);
AssertValue(KeyD, 40);
AssertValue(KeyE, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" }));
AssertValue(KeyF, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" }));
AssertDeleted(KeyX);
EnterBarrier("after-1");
}