private async Task ReplicatorDuplicatePublish() { var p1 = CreateTestProbe(_sys1); var p2 = CreateTestProbe(_sys2); var p3 = CreateTestProbe(_sys3); var probes = new[] { p1, p2, p3 }; // subscribe to updates on all 3 nodes _replicator1.Tell(Dsl.Subscribe(_keyI, p1.Ref)); _replicator2.Tell(Dsl.Subscribe(_keyI, p2.Ref)); _replicator3.Tell(Dsl.Subscribe(_keyI, p3.Ref)); // update item on 2 _replicator2.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))); Sys.Log.Info("Pushed change from sys2 for I"); // wait for write to replicate to all 3 nodes Within(TimeSpan.FromSeconds(5), () => { foreach (var p in probes) { p.ExpectMsg <Changed>(c => c.Get(_keyI).Elements.ShouldBe(ImmutableHashSet.Create("a"))); } }); // create duplicate write on node 1 Sys.Log.Info("Pushing change from sys1 for I"); _replicator1.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))); // no probe should receive an update p2.ExpectNoMsg(TimeSpan.FromSeconds(1)); }
private void UpdateORDictionaryNode2And1() { var changedProbe = CreateTestProbe(_sys2); // subscribe to updates for KeyH, then update it with a replication factor of two _replicator2.Tell(Dsl.Subscribe(_keyH, changedProbe.Ref)); _replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "a", Flag.False))); // receive local update changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.False), })).ShouldBeTrue(); // push update from node 1 _replicator1.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys1), "a", Flag.True))); // expect replication of update on node 2 changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True) })).ShouldBeTrue(); // add new value to dictionary from node 2 _replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "b", Flag.True))); changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True), new KeyValuePair <string, Flag>("b", Flag.True) })).ShouldBeTrue(); }
public void Insert_from_5_nodes_should_read_write_to_majority_when_all_nodes_connected() { var key = new ORSetKey <int>("B"); var readMajority = new ReadMajority(_timeout); var writeMajority = new WriteMajority(_timeout); RunOn(() => { var writeProbe = CreateTestProbe(); var writeAcks = MyData.Select(i => { SleepDelay(); _replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref); return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1)))); }).ToArray(); var successWriteAcks = writeAcks.OfType <UpdateSuccess>().ToArray(); var failureWriteAcks = writeAcks.OfType <IUpdateFailure>().ToArray(); successWriteAcks.Select(x => (int)x.Request).Should().BeEquivalentTo(MyData.ToArray()); successWriteAcks.Length.Should().Be(MyData.Count()); failureWriteAcks.Should().BeEmpty(); (successWriteAcks.Length + failureWriteAcks.Length).Should().Be(MyData.Count()); EnterBarrier("data-written-2"); // read from majority of nodes, which is enough to retrieve all data var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref); var result = readProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.Should().BeEquivalentTo(_expectedData); }, _nodes.ToArray()); RunOn(() => EnterBarrier("data-written-2"), Controller); EnterBarrier("after-test-2"); }
public void Insert_from_5_nodes_should_replicate_values_when_all_nodes_connected() { var key = new ORSetKey <int>("A"); RunOn(() => { var writeProbe = CreateTestProbe(); var writeAcks = MyData.Select(i => { SleepDelay(); _replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, WriteLocal.Instance, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref); return(writeProbe.ReceiveOne(TimeSpan.FromSeconds(3))); }).ToArray(); var successWriteAcks = writeAcks.OfType <UpdateSuccess>().ToArray(); var failureWriteAcks = writeAcks.OfType <IUpdateFailure>().ToArray(); successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray()); successWriteAcks.Length.Should().Be(MyData.Count()); failureWriteAcks.Should().BeEmpty(); (successWriteAcks.Length + failureWriteAcks.Length).Should().Be(MyData.Count()); // eventually all nodes will have the data Within(TimeSpan.FromSeconds(15), () => { AwaitAssert(() => { var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref); var result = readProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.Should().BeEquivalentTo(_expectedData); }); }); }, _nodes.ToArray()); EnterBarrier("after-test-1"); }
public void Cluster_CRDT_should_converge_after_many_concurrent_updates() { Within(TimeSpan.FromSeconds(10), () => { RunOn(() => { var c = GCounter.Empty; for (ulong i = 0; i < 100UL; i++) { c = c.Increment(_cluster, i); _replicator.Tell(Dsl.Update(KeyF, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1))); } var results = ReceiveN(100); results.All(x => x is UpdateSuccess).ShouldBeTrue(); }, _first, _second, _third); EnterBarrier("100-updates-done"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyF, _readTwo)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyF)).Get(KeyF); c.Value.ShouldBe(3 * 100UL); }, _first, _second, _third); EnterBarrierAfterTestStep(); }); }
public void Cluster_CRDT_should_avoid_duplicate_change_events_for_same_data() { var changedProbe = CreateTestProbe(); _replicator.Tell(Dsl.Subscribe(KeyI, changedProbe.Ref)); EnterBarrier("subscribed-I"); RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))), _second); Within(TimeSpan.FromSeconds(5), () => { var changed = changedProbe.ExpectMsg <Changed>(c => c.Get(KeyI).Elements.ShouldBe(ImmutableHashSet.Create("a"))); var keyIData = changed.Get(KeyI); Sys.Log.Debug("DEBUG: Received Changed {0}", changed); }); EnterBarrier("update-I"); RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))), _first); changedProbe.ExpectNoMsg(TimeSpan.FromSeconds(1)); EnterBarrierAfterTestStep(); }
public void Cluster_CRDT_should_work_in_2_node_cluster() { RunOn(() => { // start with 20 on both nodes _replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 20))); ExpectMsg(new UpdateSuccess(KeyB, null)); // add 1 on both nodes using WriteTwo _replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyB, null)); // the total, after replication should be 42 AwaitAssert(() => { _replicator.Tell(Dsl.Get(KeyB, _readTwo)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB); c.Value.ShouldBe(42); }); }, _first, _second); EnterBarrier("update-42"); RunOn(() => { // add 1 on both nodes using WriteAll _replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeAll, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyB, null)); // the total, after replication should be 44 AwaitAssert(() => { _replicator.Tell(Dsl.Get(KeyB, _readAll)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB); c.Value.ShouldBe(44); }); }, _first, _second); EnterBarrier("update-44"); RunOn(() => { // add 1 on both nodes using WriteMajority _replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyB, null)); // the total, after replication should be 46 AwaitAssert(() => { _replicator.Tell(Dsl.Get(KeyB, _readMajority)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB); c.Value.ShouldBe(46); }); }, _first, _second); EnterBarrierAfterTestStep(); }
/// <summary> /// On one of the nodes the data has been updated by the pruning, client can update anyway /// </summary> private void UpdateAfterPruning(int expectedValue) { _replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster.SelfUniqueAddress, 1))); ExpectMsg <Replicator.UpdateSuccess>(msg => { _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); var retrieved = ExpectMsg <Replicator.GetSuccess>().Get(KeyA); retrieved.Value.ShouldBe(expectedValue); }); }
public void Cluster_CRDT_should_support_prefer_oldest_members() { // disable gossip and delta replication to only verify the write and read operations var oldestReplicator = Sys.ActorOf( Replicator.Props( ReplicatorSettings.Create(Sys).WithPreferOldest(true).WithGossipInterval(TimeSpan.FromMinutes(1))),//.withDeltaCrdtEnabled(false)), "oldestReplicator"); Within(TimeSpan.FromSeconds(5), () => { var countProbe = CreateTestProbe(); AwaitAssert(() => { oldestReplicator.Tell(GetReplicaCount.Instance, countProbe.Ref); countProbe.ExpectMsg(new ReplicaCount(3)); }); }); EnterBarrier("oldest-replicator-started"); var probe = CreateTestProbe(); RunOn(() => { oldestReplicator.Tell( Dsl.Update(KeyK, new LWWRegister <string>(Cluster.SelfUniqueAddress, "0"), _writeTwo, a => a.WithValue(Cluster.SelfUniqueAddress, "1")), probe.Ref); probe.ExpectMsg(new UpdateSuccess(KeyK, null)); }, _second); EnterBarrier("updated-1"); RunOn(() => { // replicated to oldest oldestReplicator.Tell(new Get(KeyK, ReadLocal.Instance), probe.Ref); var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>); ((LWWRegister <string>)msg.Data).Value.Should().Be("1"); //probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1"); }, _first); RunOn(() => { // not replicated to third (not among the two oldest) oldestReplicator.Tell(Dsl.Get(KeyK, ReadLocal.Instance), probe.Ref); probe.ExpectMsg(new NotFound(KeyK, null)); // read from oldest oldestReplicator.Tell(Dsl.Get(KeyK, _readTwo), probe.Ref); var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>); ((LWWRegister <string>)msg.Data).Value.Should().Be("1"); //probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1"); }, _third); EnterBarrierAfterTestStep(); }
public void Durable_CRDT_should_work_in_a_multi_node_cluster() { Join(second, first); var r = NewReplicator(Sys); Within(TimeSpan.FromSeconds(10), () => { AwaitAssert(() => { r.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(2)); }); }); EnterBarrier("both-initialized"); r.Tell(Dsl.Update(keyA, GCounter.Empty, writeTwo, c => c.Increment(cluster))); ExpectMsg(new UpdateSuccess(keyA, null)); r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, writeTwo, c => c.Add(cluster, Myself.Name))); ExpectMsg(new UpdateSuccess(keyC, null)); EnterBarrier("update-done-" + testStepCounter); r.Tell(Dsl.Get(keyA, readTwo)); ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(2UL); r.Tell(Dsl.Get(keyC, readTwo)); ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.CreateRange(new[] { first.Name, second.Name })); EnterBarrier("values-verified-" + testStepCounter); Watch(r); Sys.Stop(r); ExpectTerminated(r); var r2 = default(IActorRef); AwaitAssert(() => r2 = NewReplicator(Sys)); // try until name is free AwaitAssert(() => { r2.Tell(Dsl.GetKeyIds); ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty); }); r2.Tell(Dsl.Get(keyA, ReadLocal.Instance)); ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(2UL); r2.Tell(Dsl.Get(keyC, ReadLocal.Instance)); ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.CreateRange(new[] { first.Name, second.Name })); EnterBarrierAfterTestStep(); }
public Updater() { var cluster = Cluster.Cluster.Get(Context.System); var replicator = DistributedData.Get(Context.System).Replicator; Receive <string>(s => { var update = Dsl.Update(Key, ORSet <string> .Empty, WriteLocal.Instance, old => old.Add(cluster.SelfUniqueAddress, s)); replicator.Tell(update); }); }
public void Durable_CRDT_should_reply_with_StoreFailure_if_store_fails() { RunOn(() => { var r = Sys.ActorOf(Replicator.Props( ReplicatorSettings.Create(Sys).WithDurableStoreProps(TestDurableStoreProps(failStore: true))), "replicator-" + testStepCounter); r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, "a", c => c.Increment(cluster))); ExpectMsg(new StoreFailure(keyA, "a")); }, first); EnterBarrierAfterTestStep(); }
private void SendUpdate(Shard.StateChange e, int retryCount) { Replicator.Tell(Dsl.Update(Key(e.EntityId), ORSet <EntryId> .Empty, _writeConsistency, Tuple.Create(e, retryCount), existing => { switch (e) { case Shard.EntityStarted started: return(existing.Add(Cluster, started.EntityId)); case Shard.EntityStopped stopped: return(existing.Remove(Cluster, stopped.EntityId)); default: throw new NotSupportedException($"DDataShard send update event not supported: {e}"); } })); }
private void TellUpdate(IActorRef replicator, ORDictionaryKey <string, PocoObject> key, IWriteConsistency consistency, PocoObject message) { var value = new PocoObject(message.TransactionId, message.Created); replicator.Tell(Dsl.Update( key, ORDictionary <string, PocoObject> .Empty, consistency, oldValue => { return(oldValue.AddOrUpdate(_cluster, value.TransactionId, null, oldTrxState => value.Merge(oldTrxState))); } )); }
public void Durable_CRDT_should_work_in_a_single_node_cluster() { Join(first, second); RunOn(() => { var r = NewReplicator(Sys); Within(TimeSpan.FromSeconds(10), () => { AwaitAssert(() => { r.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(1)); }); }); r.Tell(Dsl.Get(keyA, ReadLocal.Instance)); ExpectMsg(new NotFound(keyA, null)); r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster))); r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster))); r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster))); ExpectMsg(new UpdateSuccess(keyA, null)); ExpectMsg(new UpdateSuccess(keyA, null)); ExpectMsg(new UpdateSuccess(keyA, null)); Watch(r); Sys.Stop(r); ExpectTerminated(r); var r2 = default(IActorRef); AwaitAssert(() => r2 = NewReplicator(Sys)); // try until name is free // note that it will stash the commands until loading completed r2.Tell(Dsl.Get(keyA, ReadLocal.Instance)); ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(3UL); Watch(r2); Sys.Stop(r2); ExpectTerminated(r2); }, first); EnterBarrierAfterTestStep(); }
public void PurgeKey(string key, IWriteConsistency writeConsistency = null) { var useWriteConsistency = writeConsistency ?? WriteLocal.Instance; _receiver.Replicator.Replicator.Tell(Dsl.Update( new LWWDictionaryKey <string, DeduplicationState>(_receiver .ReplicatorKey), WriteLocal.Instance, dict => { if (dict != null) { return(dict.Remove(_receiver.SelfUniqueAddress, key)); } return(LWWDictionary <string, DeduplicationState> .Empty); })); }
public void Cluster_CRDT_should_read_repair_happens_before_GetSuccess() { RunOn(() => { _replicator.Tell(Dsl.Update(KeyG, ORSet <string> .Empty, _writeTwo, x => x .Add(_cluster, "a") .Add(_cluster, "b"))); ExpectMsg <UpdateSuccess>(); }, _first); EnterBarrier("a-b-added-to-G"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyG, _readAll)); ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyG)).Get(KeyG).Elements.SetEquals(new[] { "a", "b" }); _replicator.Tell(Dsl.Get(KeyG, ReadLocal.Instance)); ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyG)).Get(KeyG).Elements.SetEquals(new[] { "a", "b" }); }, _second); EnterBarrierAfterTestStep(); }
public void Durable_CRDT_should_be_durable_after_gossip_update() { var r = NewReplicator(); RunOn(() => { Log.Debug("sending message with sender: {}", ActorCell.GetCurrentSelfOrNoSender()); r.Tell(Dsl.Update(_keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(_cluster, Myself.Name))); ExpectMsg(new UpdateSuccess(_keyC, null)); }, _first); RunOn(() => { r.Tell(Dsl.Subscribe(_keyC, TestActor)); ExpectMsg <Changed>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name)); // must do one more roundtrip to be sure that it keyB is stored, since Changed might have // been sent out before storage TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 3)); ExpectMsg(new UpdateSuccess(_keyA, null)); Watch(r); Sys.Stop(r); ExpectTerminated(r); var r2 = default(IActorRef); AwaitAssert(() => r2 = NewReplicator()); // try until name is free AwaitAssert(() => { r2.Tell(Dsl.GetKeyIds); ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty); }); r2.Tell(Dsl.Get(_keyC, ReadLocal.Instance)); ExpectMsg <GetSuccess>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name)); }, _second); EnterBarrierAfterTestStep(); }
public void Durable_CRDT_should_be_durable_after_gossip_update() { var r = NewReplicator(Sys); RunOn(() => { r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(cluster, Myself.Name))); ExpectMsg(new UpdateSuccess(keyC, null)); }, first); RunOn(() => { r.Tell(Dsl.Subscribe(keyC, TestActor)); ExpectMsg <Changed>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name)); // must do one more roundtrip to be sure that it keyB is stored, since Changed might have // been sent out before storage r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster))); ExpectMsg(new UpdateSuccess(keyA, null)); Watch(r); Sys.Stop(r); ExpectTerminated(r); var r2 = default(IActorRef); AwaitAssert(() => r2 = NewReplicator(Sys)); AwaitAssert(() => { r2.Tell(Dsl.GetKeyIds); ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty); }); r2.Tell(Dsl.Get(keyC, ReadLocal.Instance)); ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name)); }, second); EnterBarrierAfterTestStep(); }
public void Cluster_CRDT_should_check_that_remote_update_and_local_update_both_cause_a_change_event_to_emit_with_the_merged_data() { var changedProbe = CreateTestProbe(); RunOn(() => { _replicator.Tell(Dsl.Subscribe(KeyH, changedProbe.Ref)); _replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.False))); ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.False), })).ShouldBeTrue(); }, _second); EnterBarrier("update-h1"); RunOn(() => { _replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.True))); }, _first); RunOn(() => { changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True) })).ShouldBeTrue(); _replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "b", Flag.True))); changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True), new KeyValuePair <string, Flag>("b", Flag.True) })).ShouldBeTrue(); }, _second); EnterBarrierAfterTestStep(); }
public void Pruning_of_CRDT_should_move_data_from_removed_node() { Join(First, First); Join(Second, First); Join(Third, First); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(3)); }); }); // we need the UniqueAddress var memberProbe = CreateTestProbe(); _cluster.Subscribe(memberProbe.Ref, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents, typeof(ClusterEvent.MemberUp)); var thirdUniqueAddress = memberProbe.FishForMessage(msg => msg is ClusterEvent.MemberUp up && up.Member.Address == Node(Third).Address) .AsInstanceOf <ClusterEvent.MemberUp>().Member.UniqueAddress; _replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 3))); ExpectMsg(new UpdateSuccess(_keyA, null)); _replicator.Tell(Dsl.Update(_keyB, ORSet <string> .Empty, new WriteAll(_timeout), x => x .Add(_cluster, "a") .Add(_cluster, "b") .Add(_cluster, "c"))); ExpectMsg(new UpdateSuccess(_keyB, null)); _replicator.Tell(Dsl.Update(_keyC, PNCounterDictionary <string> .Empty, new WriteAll(_timeout), x => x .Increment(_cluster, "x") .Increment(_cluster, "y"))); ExpectMsg(new UpdateSuccess(_keyC, null)); EnterBarrier("udpates-done"); _replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance)); var oldCounter = ExpectMsg <GetSuccess>().Get(_keyA); oldCounter.Value.Should().Be(9); _replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance)); var oldSet = ExpectMsg <GetSuccess>().Get(_keyB); oldSet.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" }); _replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance)); var oldMap = ExpectMsg <GetSuccess>().Get(_keyC); oldMap["x"].Should().Be(3); oldMap["y"].Should().Be(3); EnterBarrier("get-old"); RunOn(() => _cluster.Leave(Node(Third).Address), First); RunOn(() => { Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(2)); })); }, First, Second); EnterBarrier("third-removed"); RunOn(() => { Within(TimeSpan.FromSeconds(15), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance)); var counter = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyA)).Get(_keyA); counter.Value.ShouldBe(9UL); counter.NeedPruningFrom(thirdUniqueAddress).Should() .BeFalse($"{counter} shouldn't need pruning from {thirdUniqueAddress}"); }); }); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance)); var set = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyB)).Get(_keyB); set.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" }); set.NeedPruningFrom(thirdUniqueAddress).Should() .BeFalse($"{set} shouldn't need pruning from {thirdUniqueAddress}"); }); }); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance)); var map = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyC)).Get(_keyC); map["x"].Should().Be(3); map["y"].Should().Be(3); map.NeedPruningFrom(thirdUniqueAddress).Should() .BeFalse($"{map} shouldn't need pruning from {thirdUniqueAddress}"); }); }); }, First, Second); EnterBarrier("pruning-done"); void UpdateAfterPruning(ulong expectedValue) { // inject data from removed node to simulate bad data _replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Merge(oldCounter).Increment(_cluster, 1))); ExpectMsg <UpdateSuccess>(msg => { _replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance)); var retrieved = ExpectMsg <GetSuccess>().Get(_keyA); retrieved.Value.Should().Be(expectedValue); }); } RunOn(() => UpdateAfterPruning(expectedValue: 10), First); EnterBarrier("update-first-after-pruning"); RunOn(() => UpdateAfterPruning(expectedValue: 11), Second); EnterBarrier("update-second-after-pruning"); // after pruning performed and maxDissemination it is tombstoned // and we should still not be able to update with data from removed node ExpectNoMsg(_maxPruningDissemination.Add(TimeSpan.FromSeconds(3))); RunOn(() => UpdateAfterPruning(expectedValue: 12), First); EnterBarrier("update-first-after-tombstone"); RunOn(() => UpdateAfterPruning(expectedValue: 13), Second); EnterBarrier("update-second-after-tombstone"); EnterBarrier("after-1"); }
public void Cluster_CRDT_should_support_majority_quorum_write_and_read_with_3_nodes_with_1_unreachable() { Join(_third, _first); RunOn(() => { Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(3)); })); }, _first, _second, _third); EnterBarrier("3-nodes"); RunOn(() => { _replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 50))); ExpectMsg(new UpdateSuccess(KeyE, null)); }, _first, _second, _third); EnterBarrier("write-initial-majority"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyE, _readMajority)); var c150 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE); c150.Value.ShouldBe(150); }, _first, _second, _third); EnterBarrier("read-initial-majority"); RunOn(() => { TestConductor.Blackhole(_first, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5)); TestConductor.Blackhole(_second, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5)); }, _first); EnterBarrier("blackhole-third"); RunOn(() => { _replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyE, null)); }, _second); EnterBarrier("local-update-from-second"); RunOn(() => { // ReadMajority should retrive the previous update from second, before applying the modification var probe1 = CreateTestProbe(); var probe2 = CreateTestProbe(); _replicator.Tell(Dsl.Get(KeyE, _readMajority), probe2.Ref); probe2.ExpectMsg <GetSuccess>(); _replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, data => { probe1.Ref.Tell(data.Value); return(data.Increment(_cluster, 1)); }), probe2.Ref); // verify read your own writes, without waiting for the UpdateSuccess reply // note that the order of the replies are not defined, and therefore we use separate probes var probe3 = CreateTestProbe(); _replicator.Tell(Dsl.Get(KeyE, _readMajority), probe3.Ref); probe1.ExpectMsg(151); probe2.ExpectMsg(new UpdateSuccess(KeyE, null)); var c152 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE); c152.Value.ShouldBe(152); }, _first); EnterBarrier("majority-update-from-first"); RunOn(() => { var probe1 = CreateTestProbe(); _replicator.Tell(Dsl.Get(KeyE, _readMajority), probe1.Ref); probe1.ExpectMsg <GetSuccess>(); _replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 153, x => x.Increment(_cluster, 1)), probe1.Ref); // verify read your own writes, without waiting for the UpdateSuccess reply // note that the order of the replies are not defined, and therefore we use separate probes var probe2 = CreateTestProbe(); _replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 154, x => x.Increment(_cluster, 1)), probe2.Ref); var probe3 = CreateTestProbe(); _replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 155, x => x.Increment(_cluster, 1)), probe3.Ref); var probe5 = CreateTestProbe(); _replicator.Tell(Dsl.Get(KeyE, _readMajority), probe5.Ref); probe1.ExpectMsg(new UpdateSuccess(KeyE, 153)); probe2.ExpectMsg(new UpdateSuccess(KeyE, 154)); probe3.ExpectMsg(new UpdateSuccess(KeyE, 155)); var c155 = probe5.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE); c155.Value.ShouldBe(155); }, _second); EnterBarrier("majority-update-from-second"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyE2, _readAll, 998)); ExpectMsg(new GetFailure(KeyE2, 998), _timeOut.Add(TimeSpan.FromSeconds(1))); _replicator.Tell(Dsl.Get(KeyE2, Dsl.ReadLocal)); ExpectMsg(new NotFound(KeyE2, null)); }, _first, _second); EnterBarrier("read-all-fail-update"); RunOn(() => { TestConductor.PassThrough(_first, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5)); TestConductor.PassThrough(_second, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5)); }, _first); EnterBarrier("passThrough-third"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyE, _readMajority)); var c155 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE); c155.Value.ShouldBe(155); }, _third); EnterBarrierAfterTestStep(); }
public void Insert_from_5_nodes_should_replicate_values_after_partition() { var key = new ORSetKey <int>("C"); RunOn(() => { SleepBeforePartition(); foreach (var a in new List <RoleName> { N1, N4, N5 }) { foreach (var b in new List <RoleName> { N2, N3 }) { TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3)); } } SleepDuringPartition(); foreach (var a in new List <RoleName> { N1, N4, N5 }) { foreach (var b in new List <RoleName> { N2, N3 }) { TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3)); } } EnterBarrier("partition-healed-3"); }, Controller); RunOn(() => { var writeProbe = CreateTestProbe(); var writeAcks = MyData.Select(i => { SleepDelay(); _replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, WriteLocal.Instance, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref); return(writeProbe.ReceiveOne(TimeSpan.FromSeconds(3))); }).ToArray(); var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray(); var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray(); successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray()); successWriteAcks.Length.ShouldBe(MyData.Count()); failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]); (successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count()); EnterBarrier("partition-healed-3"); // eventually all nodes will have the data Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() => { var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref); var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.ShouldBe(_expectedData); })); }, _nodes.ToArray()); EnterBarrier("after-test-3"); }
public void Cluster_CRDT_should_be_replicated_after_successful_update() { var changedProbe = CreateTestProbe(); RunOn(() => { _replicator.Tell(Dsl.Subscribe(KeyC, changedProbe.Ref)); }, _first, _second); RunOn(() => { _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30))); ExpectMsg(new UpdateSuccess(KeyC, null)); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30); _replicator.Tell(Dsl.Update(KeyY, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30))); ExpectMsg(new UpdateSuccess(KeyY, null)); _replicator.Tell(Dsl.Update(KeyZ, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 30))); ExpectMsg(new UpdateSuccess(KeyZ, null)); }, _first); EnterBarrier("update-c30"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c30 = ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyC)).Get(KeyC); c30.Value.ShouldBe(30); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30); // replicate with gossip after WriteLocal _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyC, null)); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31); _replicator.Tell(Dsl.Delete(KeyY, WriteLocal.Instance, 777)); ExpectMsg(new DeleteSuccess(KeyY, 777)); _replicator.Tell(Dsl.Get(KeyZ, _readMajority)); ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyZ)).Get(KeyZ).Value.ShouldBe(30); }, _second); EnterBarrier("update-c31"); RunOn(() => { // KeyC and deleted KeyY should be replicated via gossip to the other node Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC); c.Value.ShouldBe(31); _replicator.Tell(Dsl.Get(KeyY, ReadLocal.Instance)); ExpectMsg(new DataDeleted(KeyY)); }); }); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31); }, _first); EnterBarrier("verified-c31"); // and also for concurrent updates RunOn(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c31 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC); c31.Value.ShouldBe(31); _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyC, null)); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC); c.Value.ShouldBe(33); }); }); }, _first, _second); EnterBarrierAfterTestStep(); }
public void Cluster_CRDT_should_converge_after_partition() { RunOn(() => { _replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 40))); ExpectMsg(new UpdateSuccess(KeyD, null)); TestConductor.Blackhole(_first, _second, ThrottleTransportAdapter.Direction.Both) .Wait(TimeSpan.FromSeconds(10)); }, _first); EnterBarrier("blackhole-first-second"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyD, ReadLocal.Instance)); var c40 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyD)).Get(KeyD); c40.Value.ShouldBe(40); _replicator.Tell(Dsl.Update(KeyD, GCounter.Empty.Increment(_cluster, 1), _writeTwo, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateTimeout(KeyD, null), _timeOut.Add(TimeSpan.FromSeconds(1))); _replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateTimeout(KeyD, null), _timeOut.Add(TimeSpan.FromSeconds(1))); }, _first, _second); RunOn(() => { //TODO: for some reason this RunOn never gets called for (int i = 1; i <= 30; i++) { var n = i; var keydn = new GCounterKey("D" + n); _replicator.Tell(Dsl.Update(keydn, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, n))); ExpectMsg(new UpdateSuccess(keydn, null)); } }, _first); EnterBarrier("updates-during-partion"); RunOn(() => { TestConductor.PassThrough(_first, _second, ThrottleTransportAdapter.Direction.Both) .Wait(TimeSpan.FromSeconds(5)); }, _first); EnterBarrier("passThrough-first-second"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyD, _readTwo)); var c44 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyD)).Get(KeyD); c44.Value.ShouldBe(44); Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() => { for (int i = 1; i <= 30; i++) { var keydn = new GCounterKey("D" + i); _replicator.Tell(Dsl.Get(keydn, ReadLocal.Instance)); ExpectMsg <GetSuccess>(g => Equals(g.Key, keydn)).Get(keydn).Value.ShouldBe(i); } })); }, _first, _second); EnterBarrierAfterTestStep(); }
public void Insert_from_5_nodes_should_write_to_majority_during_3_and_2_partition_and_read_from_majority_after_partition() { var key = new ORSetKey <int>("D"); var readMajority = new ReadMajority(_timeout); var writeMajority = new WriteMajority(_timeout); RunOn(() => { SleepBeforePartition(); foreach (var a in new List <RoleName> { N1, N4, N5 }) { foreach (var b in new List <RoleName> { N2, N3 }) { TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3)); } } SleepDuringPartition(); foreach (var a in new List <RoleName> { N1, N4, N5 }) { foreach (var b in new List <RoleName> { N2, N3 }) { TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3)); } } EnterBarrier("partition-healed-4"); }, Controller); RunOn(() => { var writeProbe = CreateTestProbe(); var writeAcks = MyData.Select(i => { SleepDelay(); _replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref); return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1)))); }).ToArray(); var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray(); var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray(); RunOn(() => { successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray()); successWriteAcks.Length.ShouldBe(MyData.Count()); failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]); }, N1, N4, N5); RunOn(() => { // without delays all could teoretically have been written before the blackhole if (_delayMillis != 0) { failureWriteAcks.ShouldNotBe(new Replicator.IUpdateFailure[0]); } }, N2, N3); (successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count()); EnterBarrier("partition-healed-4"); // on the 2 node side, read from majority of nodes is enough to read all writes RunOn(() => { var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref); var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.ShouldBe(_expectedData); }, N2, N3); // but on the 3 node side, read from majority doesn't mean that we are guaranteed to see // the writes from the other side, yet // eventually all nodes will have the data Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() => { var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref); var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.ShouldBe(_expectedData); })); }, _nodes.ToArray()); EnterBarrier("after-test-4"); }
public void Cluster_CRDT_should_work_in_single_node_cluster() { Join(_first, _first); RunOn(() => { Within(TimeSpan.FromSeconds(5.0), () => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(1)); }); var changedProbe = CreateTestProbe(); _replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref)); _replicator.Tell(Dsl.Subscribe(KeyX, changedProbe.Ref)); _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); ExpectMsg(new NotFound(KeyA, null)); var c3 = GCounter.Empty.Increment(_cluster, 3); var update = Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 3)); _replicator.Tell(update); ExpectMsg(new UpdateSuccess(KeyA, null)); _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); ExpectMsg(new GetSuccess(KeyA, null, c3)); changedProbe.ExpectMsg(new Changed(KeyA, c3)); var changedProbe2 = CreateTestProbe(); _replicator.Tell(new Subscribe(KeyA, changedProbe2.Ref)); changedProbe2.ExpectMsg(new Changed(KeyA, c3)); var c4 = c3.Increment(_cluster); // too strong consistency level _replicator.Tell(Dsl.Update(KeyA, _writeTwo, x => x.Increment(_cluster))); ExpectMsg(new UpdateTimeout(KeyA, null), _timeOut.Add(TimeSpan.FromSeconds(1))); _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); ExpectMsg(new GetSuccess(KeyA, null, c4)); changedProbe.ExpectMsg(new Changed(KeyA, c4)); var c5 = c4.Increment(_cluster); // too strong consistency level _replicator.Tell(Dsl.Update(KeyA, _writeMajority, x => x.Increment(_cluster))); ExpectMsg(new UpdateSuccess(KeyA, null)); _replicator.Tell(Dsl.Get(KeyA, _readMajority)); ExpectMsg(new GetSuccess(KeyA, null, c5)); changedProbe.ExpectMsg(new Changed(KeyA, c5)); var c6 = c5.Increment(_cluster); _replicator.Tell(Dsl.Update(KeyA, _writeAll, x => x.Increment(_cluster))); ExpectMsg(new UpdateSuccess(KeyA, null)); _replicator.Tell(Dsl.Get(KeyA, _readAll)); ExpectMsg(new GetSuccess(KeyA, null, c6)); changedProbe.ExpectMsg(new Changed(KeyA, c6)); var c9 = GCounter.Empty.Increment(_cluster, 9); _replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 9))); ExpectMsg(new UpdateSuccess(KeyX, null)); changedProbe.ExpectMsg(new Changed(KeyX, c9)); _replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance)); ExpectMsg(new DeleteSuccess(KeyX)); changedProbe.ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Get(KeyX, ReadLocal.Instance)); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Get(KeyX, _readAll)); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Update(KeyX, WriteLocal.Instance, x => x.Increment(_cluster))); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance)); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.GetKeyIds); ExpectMsg(new GetKeysIdsResult(ImmutableHashSet <string> .Empty.Add("A"))); }, _first); EnterBarrierAfterTestStep(); }
public void Pruning_of_durable_CRDT_should_move_data_from_removed_node() { Join(first, first); Join(second, first); var sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config); var cluster2 = Akka.Cluster.Cluster.Get(sys2); var replicator2 = StartReplicator(sys2); var probe2 = new TestProbe(sys2, new XunitAssertions()); cluster2.Join(Node(first).Address); Within(TimeSpan.FromSeconds(5), () => AwaitAssert(() => { replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(4)); replicator2.Tell(Dsl.GetReplicaCount, probe2.Ref); probe2.ExpectMsg(new ReplicaCount(4)); })); replicator.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster))); ExpectMsg(new UpdateSuccess(keyA, null)); replicator2.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster2, 2)), probe2.Ref); probe2.ExpectMsg(new UpdateSuccess(keyA, null)); EnterBarrier("updates-done"); Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() => { replicator.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1)))); var counter1 = ExpectMsg <GetSuccess>().Get(keyA); counter1.Value.ShouldBe(10UL); counter1.State.Count.ShouldBe(4); })); Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() => { replicator2.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))), probe2.Ref); var counter2 = probe2.ExpectMsg <GetSuccess>().Get(keyA); counter2.Value.ShouldBe(10UL); counter2.State.Count.ShouldBe(4); })); EnterBarrier("get1"); RunOn(() => cluster.Leave(cluster2.SelfAddress), first); Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() => { replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(3)); })); EnterBarrier("removed"); RunOn(() => sys2.Terminate().Wait(TimeSpan.FromSeconds(5)), first); Within(TimeSpan.FromSeconds(15), () => { var values = ImmutableHashSet <int> .Empty; AwaitAssert(() => { replicator.Tell(Dsl.Get(keyA, ReadLocal.Instance)); var counter3 = ExpectMsg <GetSuccess>().Get(keyA); var value = counter3.Value; values = values.Add((int)value); value.ShouldBe(10UL); counter3.State.Count.ShouldBe(3); }); values.ShouldBe(ImmutableHashSet.Create(10)); }); EnterBarrier("prunned"); RunOn(() => { var addr = cluster2.SelfAddress; var sys3 = ActorSystem.Create(Sys.Name, ConfigurationFactory.ParseString(@" ").WithFallback(Sys.Settings.Config)); var cluster3 = Akka.Cluster.Cluster.Get(sys3); var replicator3 = StartReplicator(sys3); var probe3 = new TestProbe(sys3, new XunitAssertions()); cluster3.Join(Node(first).Address); Within(TimeSpan.FromSeconds(10), () => { var values = ImmutableHashSet <int> .Empty; AwaitAssert(() => { replicator3.Tell(Dsl.Get(keyA, ReadLocal.Instance), probe3.Ref); var counter4 = probe3.ExpectMsg <GetSuccess>().Get(keyA); var value = counter4.Value; values.Add((int)value); value.ShouldBe(10UL); counter4.State.Count.ShouldBe(3); }); values.ShouldBe(ImmutableHashSet.Create(10)); }); // after merging with others replicator3.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault))); var counter5 = ExpectMsg <GetSuccess>().Get(keyA); counter5.Value.ShouldBe(10UL); counter5.State.Count.ShouldBe(3); }, first); EnterBarrier("sys3-started"); replicator.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault))); var counter6 = ExpectMsg <GetSuccess>().Get(keyA); counter6.Value.ShouldBe(10UL); counter6.State.Count.ShouldBe(3); EnterBarrier("after-1"); }
public void Replicator_in_chaotic_cluster_should_replicate_data_in_initial_phase() { Join(First, First); Join(Second, First); Join(Third, First); Join(Fourth, First); Join(Fifth, First); Within(TimeSpan.FromSeconds(10), () => { AwaitAssert(() => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(5)); }); }); RunOn(() => { for (int i = 0; i < 5; i++) { _replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1))); _replicator.Tell(Dsl.Update(KeyB, PNCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1))); _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 1))); } ReceiveN(15).Select(x => x.GetType()).ToImmutableHashSet().ShouldBe(new[] { typeof(UpdateSuccess) }); }, First); RunOn(() => { _replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 20))); _replicator.Tell(Dsl.Update(KeyB, PNCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 20))); _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 20))); ReceiveN(3).ToImmutableHashSet().ShouldBe(new[] { new UpdateSuccess(KeyA, null), new UpdateSuccess(KeyB, null), new UpdateSuccess(KeyC, null) }); _replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, WriteLocal.Instance, x => x.Add("e1").Add("e2"))); ExpectMsg(new UpdateSuccess(KeyE, null)); _replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, WriteLocal.Instance, x => x .Add(_cluster, "e1") .Add(_cluster, "e2"))); ExpectMsg(new UpdateSuccess(KeyF, null)); }, Second); RunOn(() => { _replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 40))); ExpectMsg(new UpdateSuccess(KeyD, null)); _replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, WriteLocal.Instance, x => x.Add("e2").Add("e3"))); ExpectMsg(new UpdateSuccess(KeyE, null)); _replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, WriteLocal.Instance, x => x .Add(_cluster, "e2") .Add(_cluster, "e3"))); ExpectMsg(new UpdateSuccess(KeyF, null)); }, Fourth); RunOn(() => { _replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 50))); ExpectMsg(new UpdateSuccess(KeyX, null)); _replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance)); ExpectMsg(new DeleteSuccess(KeyX)); }, Fifth); EnterBarrier("initial-updates-done"); AssertValue(KeyA, 25); AssertValue(KeyB, 15); AssertValue(KeyC, 25); AssertValue(KeyD, 40); AssertValue(KeyE, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" })); AssertValue(KeyF, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" })); AssertDeleted(KeyX); EnterBarrier("after-1"); }
public void Replicator_in_chaotic_cluster_should_be_available_during_network_split() { var side1 = new[] { First, Second }; var side2 = new[] { Third, Fourth, Fifth }; RunOn(() => { foreach (var a in side1) { foreach (var b in side2) { TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(1)); } } }, First); EnterBarrier("split"); RunOn(() => { _replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyA, null)); }, First); RunOn(() => { _replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 2))); ExpectMsg(new UpdateSuccess(KeyA, null)); _replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, new WriteTo(2, _timeout), x => x.Add("e4"))); ExpectMsg(new UpdateSuccess(KeyE, null)); _replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, new WriteTo(2, _timeout), x => x.Remove(_cluster, "e2"))); ExpectMsg(new UpdateSuccess(KeyF, null)); }, Third); RunOn(() => { _replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyD, null)); }, Fourth); EnterBarrier("update-during-split"); RunOn(() => { AssertValue(KeyA, 26); AssertValue(KeyB, 15); AssertValue(KeyD, 41); AssertValue(KeyE, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" })); AssertValue(KeyF, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" })); }, side1); RunOn(() => { AssertValue(KeyA, 27); AssertValue(KeyB, 15); AssertValue(KeyD, 41); AssertValue(KeyE, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3", "e4" })); AssertValue(KeyF, ImmutableHashSet.CreateRange(new[] { "e1", "e3" })); }, side2); EnterBarrier("update-durin-split-verified"); RunOn(() => TestConductor.Exit(Fourth, 0).Wait(TimeSpan.FromSeconds(5)), First); EnterBarrier("after-2"); }