public void Pruning_of_durable_CRDT_should_move_data_from_removed_node()
{
Join(first, first);
Join(second, first);
var sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
var cluster2 = Akka.Cluster.Cluster.Get(sys2);
var replicator2 = StartReplicator(sys2);
var probe2 = new TestProbe(sys2, new XunitAssertions());
cluster2.Join(Node(first).Address);
Within(TimeSpan.FromSeconds(5), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(4));
replicator2.Tell(Dsl.GetReplicaCount, probe2.Ref);
probe2.ExpectMsg(new ReplicaCount(4));
}));
replicator.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
replicator2.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster2, 2)), probe2.Ref);
probe2.ExpectMsg(new UpdateSuccess(keyA, null));
EnterBarrier("updates-done");
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))));
var counter1 = ExpectMsg <GetSuccess>().Get(keyA);
counter1.Value.ShouldBe(10UL);
counter1.State.Count.ShouldBe(4);
}));
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
replicator2.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))), probe2.Ref);
var counter2 = probe2.ExpectMsg <GetSuccess>().Get(keyA);
counter2.Value.ShouldBe(10UL);
counter2.State.Count.ShouldBe(4);
}));
EnterBarrier("get1");
RunOn(() => cluster.Leave(cluster2.SelfAddress), first);
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(3));
}));
EnterBarrier("removed");
RunOn(() => sys2.Terminate().Wait(TimeSpan.FromSeconds(5)), first);
Within(TimeSpan.FromSeconds(15), () =>
{
var values = ImmutableHashSet <int> .Empty;
AwaitAssert(() =>
{
replicator.Tell(Dsl.Get(keyA, ReadLocal.Instance));
var counter3 = ExpectMsg <GetSuccess>().Get(keyA);
var value = counter3.Value;
values = values.Add((int)value);
value.ShouldBe(10UL);
counter3.State.Count.ShouldBe(3);
});
values.ShouldBe(ImmutableHashSet.Create(10));
});
EnterBarrier("prunned");
RunOn(() =>
{
var addr = cluster2.SelfAddress;
var sys3 = ActorSystem.Create(Sys.Name, ConfigurationFactory.ParseString(@"
").WithFallback(Sys.Settings.Config));
var cluster3 = Akka.Cluster.Cluster.Get(sys3);
var replicator3 = StartReplicator(sys3);
var probe3 = new TestProbe(sys3, new XunitAssertions());
cluster3.Join(Node(first).Address);
Within(TimeSpan.FromSeconds(10), () =>
{
var values = ImmutableHashSet <int> .Empty;
AwaitAssert(() =>
{
replicator3.Tell(Dsl.Get(keyA, ReadLocal.Instance), probe3.Ref);
var counter4 = probe3.ExpectMsg <GetSuccess>().Get(keyA);
var value = counter4.Value;
values.Add((int)value);
value.ShouldBe(10UL);
counter4.State.Count.ShouldBe(3);
});
values.ShouldBe(ImmutableHashSet.Create(10));
});
// after merging with others
replicator3.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault)));
var counter5 = ExpectMsg <GetSuccess>().Get(keyA);
counter5.Value.ShouldBe(10UL);
counter5.State.Count.ShouldBe(3);
}, first);
EnterBarrier("sys3-started");
replicator.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault)));
var counter6 = ExpectMsg <GetSuccess>().Get(keyA);
counter6.Value.ShouldBe(10UL);
counter6.State.Count.ShouldBe(3);
EnterBarrier("after-1");
}
public void Pruning_of_CRDT_should_move_data_from_removed_node()
{
Join(First, First);
Join(Second, First);
Join(Third, First);
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(3));
});
});
// we need the UniqueAddress
var memberProbe = CreateTestProbe();
_cluster.Subscribe(memberProbe.Ref, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents,
typeof(ClusterEvent.MemberUp));
var thirdUniqueAddress = memberProbe.FishForMessage(msg =>
msg is ClusterEvent.MemberUp up && up.Member.Address == Node(Third).Address)
.AsInstanceOf <ClusterEvent.MemberUp>().Member.UniqueAddress;
_replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 3)));
ExpectMsg(new UpdateSuccess(_keyA, null));
_replicator.Tell(Dsl.Update(_keyB, ORSet <string> .Empty, new WriteAll(_timeout), x => x
.Add(_cluster, "a")
.Add(_cluster, "b")
.Add(_cluster, "c")));
ExpectMsg(new UpdateSuccess(_keyB, null));
_replicator.Tell(Dsl.Update(_keyC, PNCounterDictionary <string> .Empty, new WriteAll(_timeout), x => x
.Increment(_cluster, "x")
.Increment(_cluster, "y")));
ExpectMsg(new UpdateSuccess(_keyC, null));
EnterBarrier("udpates-done");
_replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
var oldCounter = ExpectMsg <GetSuccess>().Get(_keyA);
oldCounter.Value.Should().Be(9);
_replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance));
var oldSet = ExpectMsg <GetSuccess>().Get(_keyB);
oldSet.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" });
_replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance));
var oldMap = ExpectMsg <GetSuccess>().Get(_keyC);
oldMap["x"].Should().Be(3);
oldMap["y"].Should().Be(3);
EnterBarrier("get-old");
RunOn(() => _cluster.Leave(Node(Third).Address), First);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(2));
}));
}, First, Second);
EnterBarrier("third-removed");
RunOn(() =>
{
Within(TimeSpan.FromSeconds(15), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
var counter = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyA)).Get(_keyA);
counter.Value.ShouldBe(9UL);
counter.NeedPruningFrom(thirdUniqueAddress).Should()
.BeFalse($"{counter} shouldn't need pruning from {thirdUniqueAddress}");
});
});
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance));
var set = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyB)).Get(_keyB);
set.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" });
set.NeedPruningFrom(thirdUniqueAddress).Should()
.BeFalse($"{set} shouldn't need pruning from {thirdUniqueAddress}");
});
});
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance));
var map = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyC)).Get(_keyC);
map["x"].Should().Be(3);
map["y"].Should().Be(3);
map.NeedPruningFrom(thirdUniqueAddress).Should()
.BeFalse($"{map} shouldn't need pruning from {thirdUniqueAddress}");
});
});
}, First, Second);
EnterBarrier("pruning-done");
void UpdateAfterPruning(ulong expectedValue)
{
// inject data from removed node to simulate bad data
_replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Merge(oldCounter).Increment(_cluster, 1)));
ExpectMsg <UpdateSuccess>(msg =>
{
_replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
var retrieved = ExpectMsg <GetSuccess>().Get(_keyA);
retrieved.Value.Should().Be(expectedValue);
});
}
RunOn(() => UpdateAfterPruning(expectedValue: 10), First);
EnterBarrier("update-first-after-pruning");
RunOn(() => UpdateAfterPruning(expectedValue: 11), Second);
EnterBarrier("update-second-after-pruning");
// after pruning performed and maxDissemination it is tombstoned
// and we should still not be able to update with data from removed node
ExpectNoMsg(_maxPruningDissemination.Add(TimeSpan.FromSeconds(3)));
RunOn(() => UpdateAfterPruning(expectedValue: 12), First);
EnterBarrier("update-first-after-tombstone");
RunOn(() => UpdateAfterPruning(expectedValue: 13), Second);
EnterBarrier("update-second-after-tombstone");
EnterBarrier("after-1");
}
