public void Pruning_of_durable_CRDT_should_move_data_from_removed_node() { Join(first, first); Join(second, first); var sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config); var cluster2 = Akka.Cluster.Cluster.Get(sys2); var replicator2 = StartReplicator(sys2); var probe2 = new TestProbe(sys2, new XunitAssertions()); cluster2.Join(Node(first).Address); Within(TimeSpan.FromSeconds(5), () => AwaitAssert(() => { replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(4)); replicator2.Tell(Dsl.GetReplicaCount, probe2.Ref); probe2.ExpectMsg(new ReplicaCount(4)); })); replicator.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster))); ExpectMsg(new UpdateSuccess(keyA, null)); replicator2.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster2, 2)), probe2.Ref); probe2.ExpectMsg(new UpdateSuccess(keyA, null)); EnterBarrier("updates-done"); Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() => { replicator.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1)))); var counter1 = ExpectMsg <GetSuccess>().Get(keyA); counter1.Value.ShouldBe(10UL); counter1.State.Count.ShouldBe(4); })); Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() => { replicator2.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))), probe2.Ref); var counter2 = probe2.ExpectMsg <GetSuccess>().Get(keyA); counter2.Value.ShouldBe(10UL); counter2.State.Count.ShouldBe(4); })); EnterBarrier("get1"); RunOn(() => cluster.Leave(cluster2.SelfAddress), first); Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() => { replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(3)); })); EnterBarrier("removed"); RunOn(() => sys2.Terminate().Wait(TimeSpan.FromSeconds(5)), first); Within(TimeSpan.FromSeconds(15), () => { var values = ImmutableHashSet <int> .Empty; AwaitAssert(() => { replicator.Tell(Dsl.Get(keyA, ReadLocal.Instance)); var counter3 = ExpectMsg <GetSuccess>().Get(keyA); var value = counter3.Value; values = values.Add((int)value); value.ShouldBe(10UL); counter3.State.Count.ShouldBe(3); }); values.ShouldBe(ImmutableHashSet.Create(10)); }); EnterBarrier("prunned"); RunOn(() => { var addr = cluster2.SelfAddress; var sys3 = ActorSystem.Create(Sys.Name, ConfigurationFactory.ParseString(@" ").WithFallback(Sys.Settings.Config)); var cluster3 = Akka.Cluster.Cluster.Get(sys3); var replicator3 = StartReplicator(sys3); var probe3 = new TestProbe(sys3, new XunitAssertions()); cluster3.Join(Node(first).Address); Within(TimeSpan.FromSeconds(10), () => { var values = ImmutableHashSet <int> .Empty; AwaitAssert(() => { replicator3.Tell(Dsl.Get(keyA, ReadLocal.Instance), probe3.Ref); var counter4 = probe3.ExpectMsg <GetSuccess>().Get(keyA); var value = counter4.Value; values.Add((int)value); value.ShouldBe(10UL); counter4.State.Count.ShouldBe(3); }); values.ShouldBe(ImmutableHashSet.Create(10)); }); // after merging with others replicator3.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault))); var counter5 = ExpectMsg <GetSuccess>().Get(keyA); counter5.Value.ShouldBe(10UL); counter5.State.Count.ShouldBe(3); }, first); EnterBarrier("sys3-started"); replicator.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault))); var counter6 = ExpectMsg <GetSuccess>().Get(keyA); counter6.Value.ShouldBe(10UL); counter6.State.Count.ShouldBe(3); EnterBarrier("after-1"); }
public void Pruning_of_CRDT_should_move_data_from_removed_node() { Join(First, First); Join(Second, First); Join(Third, First); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(3)); }); }); // we need the UniqueAddress var memberProbe = CreateTestProbe(); _cluster.Subscribe(memberProbe.Ref, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents, typeof(ClusterEvent.MemberUp)); var thirdUniqueAddress = memberProbe.FishForMessage(msg => msg is ClusterEvent.MemberUp up && up.Member.Address == Node(Third).Address) .AsInstanceOf <ClusterEvent.MemberUp>().Member.UniqueAddress; _replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 3))); ExpectMsg(new UpdateSuccess(_keyA, null)); _replicator.Tell(Dsl.Update(_keyB, ORSet <string> .Empty, new WriteAll(_timeout), x => x .Add(_cluster, "a") .Add(_cluster, "b") .Add(_cluster, "c"))); ExpectMsg(new UpdateSuccess(_keyB, null)); _replicator.Tell(Dsl.Update(_keyC, PNCounterDictionary <string> .Empty, new WriteAll(_timeout), x => x .Increment(_cluster, "x") .Increment(_cluster, "y"))); ExpectMsg(new UpdateSuccess(_keyC, null)); EnterBarrier("udpates-done"); _replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance)); var oldCounter = ExpectMsg <GetSuccess>().Get(_keyA); oldCounter.Value.Should().Be(9); _replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance)); var oldSet = ExpectMsg <GetSuccess>().Get(_keyB); oldSet.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" }); _replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance)); var oldMap = ExpectMsg <GetSuccess>().Get(_keyC); oldMap["x"].Should().Be(3); oldMap["y"].Should().Be(3); EnterBarrier("get-old"); RunOn(() => _cluster.Leave(Node(Third).Address), First); RunOn(() => { Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(2)); })); }, First, Second); EnterBarrier("third-removed"); RunOn(() => { Within(TimeSpan.FromSeconds(15), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance)); var counter = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyA)).Get(_keyA); counter.Value.ShouldBe(9UL); counter.NeedPruningFrom(thirdUniqueAddress).Should() .BeFalse($"{counter} shouldn't need pruning from {thirdUniqueAddress}"); }); }); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance)); var set = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyB)).Get(_keyB); set.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" }); set.NeedPruningFrom(thirdUniqueAddress).Should() .BeFalse($"{set} shouldn't need pruning from {thirdUniqueAddress}"); }); }); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance)); var map = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyC)).Get(_keyC); map["x"].Should().Be(3); map["y"].Should().Be(3); map.NeedPruningFrom(thirdUniqueAddress).Should() .BeFalse($"{map} shouldn't need pruning from {thirdUniqueAddress}"); }); }); }, First, Second); EnterBarrier("pruning-done"); void UpdateAfterPruning(ulong expectedValue) { // inject data from removed node to simulate bad data _replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Merge(oldCounter).Increment(_cluster, 1))); ExpectMsg <UpdateSuccess>(msg => { _replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance)); var retrieved = ExpectMsg <GetSuccess>().Get(_keyA); retrieved.Value.Should().Be(expectedValue); }); } RunOn(() => UpdateAfterPruning(expectedValue: 10), First); EnterBarrier("update-first-after-pruning"); RunOn(() => UpdateAfterPruning(expectedValue: 11), Second); EnterBarrier("update-second-after-pruning"); // after pruning performed and maxDissemination it is tombstoned // and we should still not be able to update with data from removed node ExpectNoMsg(_maxPruningDissemination.Add(TimeSpan.FromSeconds(3))); RunOn(() => UpdateAfterPruning(expectedValue: 12), First); EnterBarrier("update-first-after-tombstone"); RunOn(() => UpdateAfterPruning(expectedValue: 13), Second); EnterBarrier("update-second-after-tombstone"); EnterBarrier("after-1"); }