private async Task ReplicatorDuplicatePublish()
{
var p1 = CreateTestProbe(_sys1);
var p2 = CreateTestProbe(_sys2);
var p3 = CreateTestProbe(_sys3);
var probes = new[] { p1, p2, p3 };
// subscribe to updates on all 3 nodes
_replicator1.Tell(Dsl.Subscribe(_keyI, p1.Ref));
_replicator2.Tell(Dsl.Subscribe(_keyI, p2.Ref));
_replicator3.Tell(Dsl.Subscribe(_keyI, p3.Ref));
// update item on 2
_replicator2.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a")));
Sys.Log.Info("Pushed change from sys2 for I");
// wait for write to replicate to all 3 nodes
Within(TimeSpan.FromSeconds(5), () =>
{
foreach (var p in probes)
{
p.ExpectMsg <Changed>(c => c.Get(_keyI).Elements.ShouldBe(ImmutableHashSet.Create("a")));
}
});
// create duplicate write on node 1
Sys.Log.Info("Pushing change from sys1 for I");
_replicator1.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a")));
// no probe should receive an update
p2.ExpectNoMsg(TimeSpan.FromSeconds(1));
}
private void UpdateORDictionaryNode2And1()
{
var changedProbe = CreateTestProbe(_sys2);
// subscribe to updates for KeyH, then update it with a replication factor of two
_replicator2.Tell(Dsl.Subscribe(_keyH, changedProbe.Ref));
_replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "a", Flag.False)));
// receive local update
changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.False),
})).ShouldBeTrue();
// push update from node 1
_replicator1.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys1), "a", Flag.True)));
// expect replication of update on node 2
changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True)
})).ShouldBeTrue();
// add new value to dictionary from node 2
_replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "b", Flag.True)));
changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True),
new KeyValuePair <string, Flag>("b", Flag.True)
})).ShouldBeTrue();
}
public void Insert_from_5_nodes_should_read_write_to_majority_when_all_nodes_connected()
{
var key = new ORSetKey <int>("B");
var readMajority = new ReadMajority(_timeout);
var writeMajority = new WriteMajority(_timeout);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1))));
}).ToArray();
var successWriteAcks = writeAcks.OfType <UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <IUpdateFailure>().ToArray();
successWriteAcks.Select(x => (int)x.Request).Should().BeEquivalentTo(MyData.ToArray());
successWriteAcks.Length.Should().Be(MyData.Count());
failureWriteAcks.Should().BeEmpty();
(successWriteAcks.Length + failureWriteAcks.Length).Should().Be(MyData.Count());
EnterBarrier("data-written-2");
// read from majority of nodes, which is enough to retrieve all data
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref);
var result = readProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.Should().BeEquivalentTo(_expectedData);
}, _nodes.ToArray());
RunOn(() => EnterBarrier("data-written-2"), Controller);
EnterBarrier("after-test-2");
}
public void Insert_from_5_nodes_should_replicate_values_when_all_nodes_connected()
{
var key = new ORSetKey <int>("A");
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, WriteLocal.Instance, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(TimeSpan.FromSeconds(3)));
}).ToArray();
var successWriteAcks = writeAcks.OfType <UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <IUpdateFailure>().ToArray();
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.Should().Be(MyData.Count());
failureWriteAcks.Should().BeEmpty();
(successWriteAcks.Length + failureWriteAcks.Length).Should().Be(MyData.Count());
// eventually all nodes will have the data
Within(TimeSpan.FromSeconds(15), () =>
{
AwaitAssert(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref);
var result = readProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.Should().BeEquivalentTo(_expectedData);
});
});
}, _nodes.ToArray());
EnterBarrier("after-test-1");
}
public void Cluster_CRDT_should_converge_after_many_concurrent_updates()
{
Within(TimeSpan.FromSeconds(10), () =>
{
RunOn(() =>
{
var c = GCounter.Empty;
for (ulong i = 0; i < 100UL; i++)
{
c = c.Increment(_cluster, i);
_replicator.Tell(Dsl.Update(KeyF, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1)));
}
var results = ReceiveN(100);
results.All(x => x is UpdateSuccess).ShouldBeTrue();
}, _first, _second, _third);
EnterBarrier("100-updates-done");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyF, _readTwo));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyF)).Get(KeyF);
c.Value.ShouldBe(3 * 100UL);
}, _first, _second, _third);
EnterBarrierAfterTestStep();
});
}
public void Cluster_CRDT_should_avoid_duplicate_change_events_for_same_data()
{
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyI, changedProbe.Ref));
EnterBarrier("subscribed-I");
RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))),
_second);
Within(TimeSpan.FromSeconds(5), () =>
{
var changed = changedProbe.ExpectMsg <Changed>(c =>
c.Get(KeyI).Elements.ShouldBe(ImmutableHashSet.Create("a")));
var keyIData = changed.Get(KeyI);
Sys.Log.Debug("DEBUG: Received Changed {0}", changed);
});
EnterBarrier("update-I");
RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))),
_first);
changedProbe.ExpectNoMsg(TimeSpan.FromSeconds(1));
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_work_in_2_node_cluster()
{
RunOn(() =>
{
// start with 20 on both nodes
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 20)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// add 1 on both nodes using WriteTwo
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// the total, after replication should be 42
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyB, _readTwo));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB);
c.Value.ShouldBe(42);
});
}, _first, _second);
EnterBarrier("update-42");
RunOn(() =>
{
// add 1 on both nodes using WriteAll
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeAll, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// the total, after replication should be 44
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyB, _readAll));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB);
c.Value.ShouldBe(44);
});
}, _first, _second);
EnterBarrier("update-44");
RunOn(() =>
{
// add 1 on both nodes using WriteMajority
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// the total, after replication should be 46
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyB, _readMajority));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB);
c.Value.ShouldBe(46);
});
}, _first, _second);
EnterBarrierAfterTestStep();
}
/// <summary>
/// On one of the nodes the data has been updated by the pruning, client can update anyway
/// </summary>
private void UpdateAfterPruning(int expectedValue)
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster.SelfUniqueAddress, 1)));
ExpectMsg <Replicator.UpdateSuccess>(msg =>
{
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
var retrieved = ExpectMsg <Replicator.GetSuccess>().Get(KeyA);
retrieved.Value.ShouldBe(expectedValue);
});
}
public void Cluster_CRDT_should_support_prefer_oldest_members()
{
// disable gossip and delta replication to only verify the write and read operations
var oldestReplicator = Sys.ActorOf(
Replicator.Props(
ReplicatorSettings.Create(Sys).WithPreferOldest(true).WithGossipInterval(TimeSpan.FromMinutes(1))),//.withDeltaCrdtEnabled(false)),
"oldestReplicator");
Within(TimeSpan.FromSeconds(5), () =>
{
var countProbe = CreateTestProbe();
AwaitAssert(() =>
{
oldestReplicator.Tell(GetReplicaCount.Instance, countProbe.Ref);
countProbe.ExpectMsg(new ReplicaCount(3));
});
});
EnterBarrier("oldest-replicator-started");
var probe = CreateTestProbe();
RunOn(() =>
{
oldestReplicator.Tell(
Dsl.Update(KeyK, new LWWRegister <string>(Cluster.SelfUniqueAddress, "0"), _writeTwo, a => a.WithValue(Cluster.SelfUniqueAddress, "1")),
probe.Ref);
probe.ExpectMsg(new UpdateSuccess(KeyK, null));
}, _second);
EnterBarrier("updated-1");
RunOn(() =>
{
// replicated to oldest
oldestReplicator.Tell(new Get(KeyK, ReadLocal.Instance), probe.Ref);
var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>);
((LWWRegister <string>)msg.Data).Value.Should().Be("1");
//probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1");
}, _first);
RunOn(() =>
{
// not replicated to third (not among the two oldest)
oldestReplicator.Tell(Dsl.Get(KeyK, ReadLocal.Instance), probe.Ref);
probe.ExpectMsg(new NotFound(KeyK, null));
// read from oldest
oldestReplicator.Tell(Dsl.Get(KeyK, _readTwo), probe.Ref);
var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>);
((LWWRegister <string>)msg.Data).Value.Should().Be("1");
//probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1");
}, _third);
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_work_in_a_multi_node_cluster()
{
Join(second, first);
var r = NewReplicator(Sys);
Within(TimeSpan.FromSeconds(10), () =>
{
AwaitAssert(() =>
{
r.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(2));
});
});
EnterBarrier("both-initialized");
r.Tell(Dsl.Update(keyA, GCounter.Empty, writeTwo, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, writeTwo, c => c.Add(cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(keyC, null));
EnterBarrier("update-done-" + testStepCounter);
r.Tell(Dsl.Get(keyA, readTwo));
ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(2UL);
r.Tell(Dsl.Get(keyC, readTwo));
ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.CreateRange(new[] { first.Name, second.Name }));
EnterBarrier("values-verified-" + testStepCounter);
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator(Sys)); // try until name is free
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(keyA, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(2UL);
r2.Tell(Dsl.Get(keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.CreateRange(new[] { first.Name, second.Name }));
EnterBarrierAfterTestStep();
}
public Updater()
{
var cluster = Cluster.Cluster.Get(Context.System);
var replicator = DistributedData.Get(Context.System).Replicator;
Receive <string>(s =>
{
var update = Dsl.Update(Key, ORSet <string> .Empty, WriteLocal.Instance, old => old.Add(cluster.SelfUniqueAddress, s));
replicator.Tell(update);
});
}
public void Durable_CRDT_should_reply_with_StoreFailure_if_store_fails()
{
RunOn(() =>
{
var r = Sys.ActorOf(Replicator.Props(
ReplicatorSettings.Create(Sys).WithDurableStoreProps(TestDurableStoreProps(failStore: true))),
"replicator-" + testStepCounter);
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, "a", c => c.Increment(cluster)));
ExpectMsg(new StoreFailure(keyA, "a"));
}, first);
EnterBarrierAfterTestStep();
}
private void SendUpdate(Shard.StateChange e, int retryCount)
{
Replicator.Tell(Dsl.Update(Key(e.EntityId), ORSet <EntryId> .Empty, _writeConsistency, Tuple.Create(e, retryCount),
existing =>
{
switch (e)
{
case Shard.EntityStarted started: return(existing.Add(Cluster, started.EntityId));
case Shard.EntityStopped stopped: return(existing.Remove(Cluster, stopped.EntityId));
default: throw new NotSupportedException($"DDataShard send update event not supported: {e}");
}
}));
}
private void TellUpdate(IActorRef replicator, ORDictionaryKey <string, PocoObject> key, IWriteConsistency consistency, PocoObject message)
{
var value = new PocoObject(message.TransactionId, message.Created);
replicator.Tell(Dsl.Update(
key,
ORDictionary <string, PocoObject> .Empty,
consistency,
oldValue =>
{
return(oldValue.AddOrUpdate(_cluster, value.TransactionId, null,
oldTrxState => value.Merge(oldTrxState)));
}
));
}
public void Durable_CRDT_should_work_in_a_single_node_cluster()
{
Join(first, second);
RunOn(() =>
{
var r = NewReplicator(Sys);
Within(TimeSpan.FromSeconds(10), () =>
{
AwaitAssert(() =>
{
r.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(1));
});
});
r.Tell(Dsl.Get(keyA, ReadLocal.Instance));
ExpectMsg(new NotFound(keyA, null));
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
ExpectMsg(new UpdateSuccess(keyA, null));
ExpectMsg(new UpdateSuccess(keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator(Sys)); // try until name is free
// note that it will stash the commands until loading completed
r2.Tell(Dsl.Get(keyA, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(3UL);
Watch(r2);
Sys.Stop(r2);
ExpectTerminated(r2);
}, first);
EnterBarrierAfterTestStep();
}
public void PurgeKey(string key, IWriteConsistency writeConsistency = null)
{
var useWriteConsistency = writeConsistency ?? WriteLocal.Instance;
_receiver.Replicator.Replicator.Tell(Dsl.Update(
new LWWDictionaryKey <string, DeduplicationState>(_receiver
.ReplicatorKey),
WriteLocal.Instance,
dict =>
{
if (dict != null)
{
return(dict.Remove(_receiver.SelfUniqueAddress, key));
}
return(LWWDictionary <string, DeduplicationState> .Empty);
}));
}
public void Cluster_CRDT_should_read_repair_happens_before_GetSuccess()
{
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyG, ORSet <string> .Empty, _writeTwo, x => x
.Add(_cluster, "a")
.Add(_cluster, "b")));
ExpectMsg <UpdateSuccess>();
}, _first);
EnterBarrier("a-b-added-to-G");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyG, _readAll));
ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyG)).Get(KeyG).Elements.SetEquals(new[] { "a", "b" });
_replicator.Tell(Dsl.Get(KeyG, ReadLocal.Instance));
ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyG)).Get(KeyG).Elements.SetEquals(new[] { "a", "b" });
}, _second);
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_be_durable_after_gossip_update()
{
var r = NewReplicator();
RunOn(() =>
{
Log.Debug("sending message with sender: {}", ActorCell.GetCurrentSelfOrNoSender());
r.Tell(Dsl.Update(_keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(_cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(_keyC, null));
}, _first);
RunOn(() =>
{
r.Tell(Dsl.Subscribe(_keyC, TestActor));
ExpectMsg <Changed>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name));
// must do one more roundtrip to be sure that it keyB is stored, since Changed might have
// been sent out before storage
TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 3));
ExpectMsg(new UpdateSuccess(_keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator()); // try until name is free
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(_keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name));
}, _second);
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_be_durable_after_gossip_update()
{
var r = NewReplicator(Sys);
RunOn(() =>
{
r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(keyC, null));
}, first);
RunOn(() =>
{
r.Tell(Dsl.Subscribe(keyC, TestActor));
ExpectMsg <Changed>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name));
// must do one more roundtrip to be sure that it keyB is stored, since Changed might have
// been sent out before storage
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator(Sys));
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name));
}, second);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_check_that_remote_update_and_local_update_both_cause_a_change_event_to_emit_with_the_merged_data()
{
var changedProbe = CreateTestProbe();
RunOn(() =>
{
_replicator.Tell(Dsl.Subscribe(KeyH, changedProbe.Ref));
_replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.False)));
ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.False),
})).ShouldBeTrue();
}, _second);
EnterBarrier("update-h1");
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.True)));
}, _first);
RunOn(() =>
{
changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True)
})).ShouldBeTrue();
_replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "b", Flag.True)));
changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True),
new KeyValuePair <string, Flag>("b", Flag.True)
})).ShouldBeTrue();
}, _second);
EnterBarrierAfterTestStep();
}
public void Pruning_of_CRDT_should_move_data_from_removed_node()
{
Join(First, First);
Join(Second, First);
Join(Third, First);
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(3));
});
});
// we need the UniqueAddress
var memberProbe = CreateTestProbe();
_cluster.Subscribe(memberProbe.Ref, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents,
typeof(ClusterEvent.MemberUp));
var thirdUniqueAddress = memberProbe.FishForMessage(msg =>
msg is ClusterEvent.MemberUp up && up.Member.Address == Node(Third).Address)
.AsInstanceOf <ClusterEvent.MemberUp>().Member.UniqueAddress;
_replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 3)));
ExpectMsg(new UpdateSuccess(_keyA, null));
_replicator.Tell(Dsl.Update(_keyB, ORSet <string> .Empty, new WriteAll(_timeout), x => x
.Add(_cluster, "a")
.Add(_cluster, "b")
.Add(_cluster, "c")));
ExpectMsg(new UpdateSuccess(_keyB, null));
_replicator.Tell(Dsl.Update(_keyC, PNCounterDictionary <string> .Empty, new WriteAll(_timeout), x => x
.Increment(_cluster, "x")
.Increment(_cluster, "y")));
ExpectMsg(new UpdateSuccess(_keyC, null));
EnterBarrier("udpates-done");
_replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
var oldCounter = ExpectMsg <GetSuccess>().Get(_keyA);
oldCounter.Value.Should().Be(9);
_replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance));
var oldSet = ExpectMsg <GetSuccess>().Get(_keyB);
oldSet.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" });
_replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance));
var oldMap = ExpectMsg <GetSuccess>().Get(_keyC);
oldMap["x"].Should().Be(3);
oldMap["y"].Should().Be(3);
EnterBarrier("get-old");
RunOn(() => _cluster.Leave(Node(Third).Address), First);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(2));
}));
}, First, Second);
EnterBarrier("third-removed");
RunOn(() =>
{
Within(TimeSpan.FromSeconds(15), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
var counter = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyA)).Get(_keyA);
counter.Value.ShouldBe(9UL);
counter.NeedPruningFrom(thirdUniqueAddress).Should()
.BeFalse($"{counter} shouldn't need pruning from {thirdUniqueAddress}");
});
});
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(_keyB, ReadLocal.Instance));
var set = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyB)).Get(_keyB);
set.Elements.Should().BeEquivalentTo(new[] { "c", "b", "a" });
set.NeedPruningFrom(thirdUniqueAddress).Should()
.BeFalse($"{set} shouldn't need pruning from {thirdUniqueAddress}");
});
});
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(_keyC, ReadLocal.Instance));
var map = ExpectMsg <GetSuccess>(msg => Equals(msg.Key, _keyC)).Get(_keyC);
map["x"].Should().Be(3);
map["y"].Should().Be(3);
map.NeedPruningFrom(thirdUniqueAddress).Should()
.BeFalse($"{map} shouldn't need pruning from {thirdUniqueAddress}");
});
});
}, First, Second);
EnterBarrier("pruning-done");
void UpdateAfterPruning(ulong expectedValue)
{
// inject data from removed node to simulate bad data
_replicator.Tell(Dsl.Update(_keyA, GCounter.Empty, new WriteAll(_timeout), x => x.Merge(oldCounter).Increment(_cluster, 1)));
ExpectMsg <UpdateSuccess>(msg =>
{
_replicator.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
var retrieved = ExpectMsg <GetSuccess>().Get(_keyA);
retrieved.Value.Should().Be(expectedValue);
});
}
RunOn(() => UpdateAfterPruning(expectedValue: 10), First);
EnterBarrier("update-first-after-pruning");
RunOn(() => UpdateAfterPruning(expectedValue: 11), Second);
EnterBarrier("update-second-after-pruning");
// after pruning performed and maxDissemination it is tombstoned
// and we should still not be able to update with data from removed node
ExpectNoMsg(_maxPruningDissemination.Add(TimeSpan.FromSeconds(3)));
RunOn(() => UpdateAfterPruning(expectedValue: 12), First);
EnterBarrier("update-first-after-tombstone");
RunOn(() => UpdateAfterPruning(expectedValue: 13), Second);
EnterBarrier("update-second-after-tombstone");
EnterBarrier("after-1");
}
public void Cluster_CRDT_should_support_majority_quorum_write_and_read_with_3_nodes_with_1_unreachable()
{
Join(_third, _first);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(3));
}));
}, _first, _second, _third);
EnterBarrier("3-nodes");
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 50)));
ExpectMsg(new UpdateSuccess(KeyE, null));
}, _first, _second, _third);
EnterBarrier("write-initial-majority");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyE, _readMajority));
var c150 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c150.Value.ShouldBe(150);
}, _first, _second, _third);
EnterBarrier("read-initial-majority");
RunOn(() =>
{
TestConductor.Blackhole(_first, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
TestConductor.Blackhole(_second, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
}, _first);
EnterBarrier("blackhole-third");
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyE, null));
}, _second);
EnterBarrier("local-update-from-second");
RunOn(() =>
{
// ReadMajority should retrive the previous update from second, before applying the modification
var probe1 = CreateTestProbe();
var probe2 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe2.Ref);
probe2.ExpectMsg <GetSuccess>();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, data =>
{
probe1.Ref.Tell(data.Value);
return(data.Increment(_cluster, 1));
}), probe2.Ref);
// verify read your own writes, without waiting for the UpdateSuccess reply
// note that the order of the replies are not defined, and therefore we use separate probes
var probe3 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe3.Ref);
probe1.ExpectMsg(151);
probe2.ExpectMsg(new UpdateSuccess(KeyE, null));
var c152 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c152.Value.ShouldBe(152);
}, _first);
EnterBarrier("majority-update-from-first");
RunOn(() =>
{
var probe1 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe1.Ref);
probe1.ExpectMsg <GetSuccess>();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 153, x => x.Increment(_cluster, 1)), probe1.Ref);
// verify read your own writes, without waiting for the UpdateSuccess reply
// note that the order of the replies are not defined, and therefore we use separate probes
var probe2 = CreateTestProbe();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 154, x => x.Increment(_cluster, 1)), probe2.Ref);
var probe3 = CreateTestProbe();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 155, x => x.Increment(_cluster, 1)), probe3.Ref);
var probe5 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe5.Ref);
probe1.ExpectMsg(new UpdateSuccess(KeyE, 153));
probe2.ExpectMsg(new UpdateSuccess(KeyE, 154));
probe3.ExpectMsg(new UpdateSuccess(KeyE, 155));
var c155 = probe5.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c155.Value.ShouldBe(155);
}, _second);
EnterBarrier("majority-update-from-second");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyE2, _readAll, 998));
ExpectMsg(new GetFailure(KeyE2, 998), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Get(KeyE2, Dsl.ReadLocal));
ExpectMsg(new NotFound(KeyE2, null));
}, _first, _second);
EnterBarrier("read-all-fail-update");
RunOn(() =>
{
TestConductor.PassThrough(_first, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
TestConductor.PassThrough(_second, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
}, _first);
EnterBarrier("passThrough-third");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyE, _readMajority));
var c155 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c155.Value.ShouldBe(155);
}, _third);
EnterBarrierAfterTestStep();
}
public void Insert_from_5_nodes_should_replicate_values_after_partition()
{
var key = new ORSetKey <int>("C");
RunOn(() =>
{
SleepBeforePartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
SleepDuringPartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
EnterBarrier("partition-healed-3");
}, Controller);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, WriteLocal.Instance, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(TimeSpan.FromSeconds(3)));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
EnterBarrier("partition-healed-3");
// eventually all nodes will have the data
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}));
}, _nodes.ToArray());
EnterBarrier("after-test-3");
}
public void Cluster_CRDT_should_be_replicated_after_successful_update()
{
var changedProbe = CreateTestProbe();
RunOn(() =>
{
_replicator.Tell(Dsl.Subscribe(KeyC, changedProbe.Ref));
}, _first, _second);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
_replicator.Tell(Dsl.Update(KeyY, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyY, null));
_replicator.Tell(Dsl.Update(KeyZ, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyZ, null));
}, _first);
EnterBarrier("update-c30");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c30 = ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyC)).Get(KeyC);
c30.Value.ShouldBe(30);
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
// replicate with gossip after WriteLocal
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
_replicator.Tell(Dsl.Delete(KeyY, WriteLocal.Instance, 777));
ExpectMsg(new DeleteSuccess(KeyY, 777));
_replicator.Tell(Dsl.Get(KeyZ, _readMajority));
ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyZ)).Get(KeyZ).Value.ShouldBe(30);
}, _second);
EnterBarrier("update-c31");
RunOn(() =>
{
// KeyC and deleted KeyY should be replicated via gossip to the other node
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(31);
_replicator.Tell(Dsl.Get(KeyY, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyY));
});
});
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
}, _first);
EnterBarrier("verified-c31");
// and also for concurrent updates
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c31 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c31.Value.ShouldBe(31);
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(33);
});
});
}, _first, _second);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_converge_after_partition()
{
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 40)));
ExpectMsg(new UpdateSuccess(KeyD, null));
TestConductor.Blackhole(_first, _second, ThrottleTransportAdapter.Direction.Both)
.Wait(TimeSpan.FromSeconds(10));
}, _first);
EnterBarrier("blackhole-first-second");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyD, ReadLocal.Instance));
var c40 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyD)).Get(KeyD);
c40.Value.ShouldBe(40);
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty.Increment(_cluster, 1), _writeTwo, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateTimeout(KeyD, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateTimeout(KeyD, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
}, _first, _second);
RunOn(() =>
{
//TODO: for some reason this RunOn never gets called
for (int i = 1; i <= 30; i++)
{
var n = i;
var keydn = new GCounterKey("D" + n);
_replicator.Tell(Dsl.Update(keydn, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, n)));
ExpectMsg(new UpdateSuccess(keydn, null));
}
}, _first);
EnterBarrier("updates-during-partion");
RunOn(() =>
{
TestConductor.PassThrough(_first, _second, ThrottleTransportAdapter.Direction.Both)
.Wait(TimeSpan.FromSeconds(5));
}, _first);
EnterBarrier("passThrough-first-second");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyD, _readTwo));
var c44 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyD)).Get(KeyD);
c44.Value.ShouldBe(44);
Within(TimeSpan.FromSeconds(10), () =>
AwaitAssert(() =>
{
for (int i = 1; i <= 30; i++)
{
var keydn = new GCounterKey("D" + i);
_replicator.Tell(Dsl.Get(keydn, ReadLocal.Instance));
ExpectMsg <GetSuccess>(g => Equals(g.Key, keydn)).Get(keydn).Value.ShouldBe(i);
}
}));
}, _first, _second);
EnterBarrierAfterTestStep();
}
public void Insert_from_5_nodes_should_write_to_majority_during_3_and_2_partition_and_read_from_majority_after_partition()
{
var key = new ORSetKey <int>("D");
var readMajority = new ReadMajority(_timeout);
var writeMajority = new WriteMajority(_timeout);
RunOn(() =>
{
SleepBeforePartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
SleepDuringPartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
EnterBarrier("partition-healed-4");
}, Controller);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1))));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
RunOn(() =>
{
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
}, N1, N4, N5);
RunOn(() =>
{
// without delays all could teoretically have been written before the blackhole
if (_delayMillis != 0)
{
failureWriteAcks.ShouldNotBe(new Replicator.IUpdateFailure[0]);
}
}, N2, N3);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
EnterBarrier("partition-healed-4");
// on the 2 node side, read from majority of nodes is enough to read all writes
RunOn(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}, N2, N3);
// but on the 3 node side, read from majority doesn't mean that we are guaranteed to see
// the writes from the other side, yet
// eventually all nodes will have the data
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}));
}, _nodes.ToArray());
EnterBarrier("after-test-4");
}
public void Cluster_CRDT_should_work_in_single_node_cluster()
{
Join(_first, _first);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(5.0), () =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(1));
});
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref));
_replicator.Tell(Dsl.Subscribe(KeyX, changedProbe.Ref));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new NotFound(KeyA, null));
var c3 = GCounter.Empty.Increment(_cluster, 3);
var update = Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 3));
_replicator.Tell(update);
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c3));
changedProbe.ExpectMsg(new Changed(KeyA, c3));
var changedProbe2 = CreateTestProbe();
_replicator.Tell(new Subscribe(KeyA, changedProbe2.Ref));
changedProbe2.ExpectMsg(new Changed(KeyA, c3));
var c4 = c3.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeTwo, x => x.Increment(_cluster)));
ExpectMsg(new UpdateTimeout(KeyA, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c4));
changedProbe.ExpectMsg(new Changed(KeyA, c4));
var c5 = c4.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeMajority, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readMajority));
ExpectMsg(new GetSuccess(KeyA, null, c5));
changedProbe.ExpectMsg(new Changed(KeyA, c5));
var c6 = c5.Increment(_cluster);
_replicator.Tell(Dsl.Update(KeyA, _writeAll, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readAll));
ExpectMsg(new GetSuccess(KeyA, null, c6));
changedProbe.ExpectMsg(new Changed(KeyA, c6));
var c9 = GCounter.Empty.Increment(_cluster, 9);
_replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 9)));
ExpectMsg(new UpdateSuccess(KeyX, null));
changedProbe.ExpectMsg(new Changed(KeyX, c9));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DeleteSuccess(KeyX));
changedProbe.ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, _readAll));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Update(KeyX, WriteLocal.Instance, x => x.Increment(_cluster)));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.GetKeyIds);
ExpectMsg(new GetKeysIdsResult(ImmutableHashSet <string> .Empty.Add("A")));
}, _first);
EnterBarrierAfterTestStep();
}
public void Pruning_of_durable_CRDT_should_move_data_from_removed_node()
{
Join(first, first);
Join(second, first);
var sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
var cluster2 = Akka.Cluster.Cluster.Get(sys2);
var replicator2 = StartReplicator(sys2);
var probe2 = new TestProbe(sys2, new XunitAssertions());
cluster2.Join(Node(first).Address);
Within(TimeSpan.FromSeconds(5), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(4));
replicator2.Tell(Dsl.GetReplicaCount, probe2.Ref);
probe2.ExpectMsg(new ReplicaCount(4));
}));
replicator.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
replicator2.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster2, 2)), probe2.Ref);
probe2.ExpectMsg(new UpdateSuccess(keyA, null));
EnterBarrier("updates-done");
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))));
var counter1 = ExpectMsg <GetSuccess>().Get(keyA);
counter1.Value.ShouldBe(10UL);
counter1.State.Count.ShouldBe(4);
}));
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
replicator2.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))), probe2.Ref);
var counter2 = probe2.ExpectMsg <GetSuccess>().Get(keyA);
counter2.Value.ShouldBe(10UL);
counter2.State.Count.ShouldBe(4);
}));
EnterBarrier("get1");
RunOn(() => cluster.Leave(cluster2.SelfAddress), first);
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(3));
}));
EnterBarrier("removed");
RunOn(() => sys2.Terminate().Wait(TimeSpan.FromSeconds(5)), first);
Within(TimeSpan.FromSeconds(15), () =>
{
var values = ImmutableHashSet <int> .Empty;
AwaitAssert(() =>
{
replicator.Tell(Dsl.Get(keyA, ReadLocal.Instance));
var counter3 = ExpectMsg <GetSuccess>().Get(keyA);
var value = counter3.Value;
values = values.Add((int)value);
value.ShouldBe(10UL);
counter3.State.Count.ShouldBe(3);
});
values.ShouldBe(ImmutableHashSet.Create(10));
});
EnterBarrier("prunned");
RunOn(() =>
{
var addr = cluster2.SelfAddress;
var sys3 = ActorSystem.Create(Sys.Name, ConfigurationFactory.ParseString(@"
").WithFallback(Sys.Settings.Config));
var cluster3 = Akka.Cluster.Cluster.Get(sys3);
var replicator3 = StartReplicator(sys3);
var probe3 = new TestProbe(sys3, new XunitAssertions());
cluster3.Join(Node(first).Address);
Within(TimeSpan.FromSeconds(10), () =>
{
var values = ImmutableHashSet <int> .Empty;
AwaitAssert(() =>
{
replicator3.Tell(Dsl.Get(keyA, ReadLocal.Instance), probe3.Ref);
var counter4 = probe3.ExpectMsg <GetSuccess>().Get(keyA);
var value = counter4.Value;
values.Add((int)value);
value.ShouldBe(10UL);
counter4.State.Count.ShouldBe(3);
});
values.ShouldBe(ImmutableHashSet.Create(10));
});
// after merging with others
replicator3.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault)));
var counter5 = ExpectMsg <GetSuccess>().Get(keyA);
counter5.Value.ShouldBe(10UL);
counter5.State.Count.ShouldBe(3);
}, first);
EnterBarrier("sys3-started");
replicator.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault)));
var counter6 = ExpectMsg <GetSuccess>().Get(keyA);
counter6.Value.ShouldBe(10UL);
counter6.State.Count.ShouldBe(3);
EnterBarrier("after-1");
}
public void Replicator_in_chaotic_cluster_should_replicate_data_in_initial_phase()
{
Join(First, First);
Join(Second, First);
Join(Third, First);
Join(Fourth, First);
Join(Fifth, First);
Within(TimeSpan.FromSeconds(10), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(5));
});
});
RunOn(() =>
{
for (int i = 0; i < 5; i++)
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
_replicator.Tell(Dsl.Update(KeyB, PNCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 1)));
}
ReceiveN(15).Select(x => x.GetType()).ToImmutableHashSet().ShouldBe(new[] { typeof(UpdateSuccess) });
}, First);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 20)));
_replicator.Tell(Dsl.Update(KeyB, PNCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 20)));
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster, 20)));
ReceiveN(3).ToImmutableHashSet().ShouldBe(new[]
{
new UpdateSuccess(KeyA, null),
new UpdateSuccess(KeyB, null),
new UpdateSuccess(KeyC, null)
});
_replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, WriteLocal.Instance, x => x.Add("e1").Add("e2")));
ExpectMsg(new UpdateSuccess(KeyE, null));
_replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, WriteLocal.Instance, x => x
.Add(_cluster, "e1")
.Add(_cluster, "e2")));
ExpectMsg(new UpdateSuccess(KeyF, null));
}, Second);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 40)));
ExpectMsg(new UpdateSuccess(KeyD, null));
_replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, WriteLocal.Instance, x => x.Add("e2").Add("e3")));
ExpectMsg(new UpdateSuccess(KeyE, null));
_replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, WriteLocal.Instance, x => x
.Add(_cluster, "e2")
.Add(_cluster, "e3")));
ExpectMsg(new UpdateSuccess(KeyF, null));
}, Fourth);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 50)));
ExpectMsg(new UpdateSuccess(KeyX, null));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DeleteSuccess(KeyX));
}, Fifth);
EnterBarrier("initial-updates-done");
AssertValue(KeyA, 25);
AssertValue(KeyB, 15);
AssertValue(KeyC, 25);
AssertValue(KeyD, 40);
AssertValue(KeyE, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" }));
AssertValue(KeyF, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" }));
AssertDeleted(KeyX);
EnterBarrier("after-1");
}
public void Replicator_in_chaotic_cluster_should_be_available_during_network_split()
{
var side1 = new[] { First, Second };
var side2 = new[] { Third, Fourth, Fifth };
RunOn(() =>
{
foreach (var a in side1)
{
foreach (var b in side2)
{
TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(1));
}
}
}, First);
EnterBarrier("split");
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyA, null));
}, First);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 2)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Update(KeyE, GSet <string> .Empty, new WriteTo(2, _timeout), x => x.Add("e4")));
ExpectMsg(new UpdateSuccess(KeyE, null));
_replicator.Tell(Dsl.Update(KeyF, ORSet <string> .Empty, new WriteTo(2, _timeout), x => x.Remove(_cluster, "e2")));
ExpectMsg(new UpdateSuccess(KeyF, null));
}, Third);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, new WriteTo(2, _timeout), x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyD, null));
}, Fourth);
EnterBarrier("update-during-split");
RunOn(() =>
{
AssertValue(KeyA, 26);
AssertValue(KeyB, 15);
AssertValue(KeyD, 41);
AssertValue(KeyE, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" }));
AssertValue(KeyF, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3" }));
}, side1);
RunOn(() =>
{
AssertValue(KeyA, 27);
AssertValue(KeyB, 15);
AssertValue(KeyD, 41);
AssertValue(KeyE, ImmutableHashSet.CreateRange(new[] { "e1", "e2", "e3", "e4" }));
AssertValue(KeyF, ImmutableHashSet.CreateRange(new[] { "e1", "e3" }));
}, side2);
EnterBarrier("update-durin-split-verified");
RunOn(() => TestConductor.Exit(Fourth, 0).Wait(TimeSpan.FromSeconds(5)), First);
EnterBarrier("after-2");
}