public void Insert_from_5_nodes_should_read_write_to_majority_when_all_nodes_connected()
{
var key = new ORSetKey <int>("B");
var readMajority = new ReadMajority(_timeout);
var writeMajority = new WriteMajority(_timeout);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1))));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
EnterBarrier("data-written-2");
// read from majority of nodes, which is enough to retrieve all data
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}, _nodes.ToArray());
RunOn(() => EnterBarrier("data-written-2"), Controller);
EnterBarrier("after-test-2");
}
private void HandleMessage(CreateEventMarket message)
{
try
{
var cluster = Cluster.Get(Context.System);
var replicator = DistributedData.Get(Context.System).Replicator;
var key = new ORSetKey <string>($"Event-{message.EventId}");
var writeConsistency = new WriteMajority(TimeSpan.FromSeconds(2));
replicator.Tell(Dsl.Update(key, ORSet <string> .Empty, writeConsistency,
existing => existing.Add(cluster, message.Market)));
var localEvent =
replicator.Ask <IGetResponse>(Dsl.Get(key, ReadLocal.Instance));
Sender.Tell(localEvent.Result);
}
catch (Exception e)
{
_log.Error(e, "Unable to process message CreateEventMarket for Event {0} Market {1}", message.EventId,
message.Market);
Sender.Tell(
$"Unable to process message CreateEventMarket for Event {message.EventId} Market {message.Market}");
}
}
public ReplicatorSpec(ReplicatorSpecConfig config)
: base(config)
{
_config = config;
_cluster = Cluster.Cluster.Get(Sys);
var settings = ReplicatorSettings.Create(Sys).WithGossipInterval(TimeSpan.FromSeconds(1.0)).WithMaxDeltaElements(10);
var props = Replicator.Props(settings);
_replicator = Sys.ActorOf(props, "replicator");
_timeOut = Dilated(TimeSpan.FromSeconds(2.0));
_writeTwo = new WriteTo(2, _timeOut);
_writeMajority = new WriteMajority(_timeOut);
_writeAll = new WriteAll(_timeOut);
_readTwo = new ReadFrom(2, _timeOut);
_readMajority = new ReadMajority(_timeOut);
_readAll = new ReadAll(_timeOut);
}
protected ReplicatorSpec(ReplicatorSpecConfig config)
: base(config, typeof(ReplicatorSpec))
{
_config = config;
_first = config.First;
_second = config.Second;
_third = config.Third;
_cluster = Akka.Cluster.Cluster.Get(Sys);
var settings = ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1.0))
.WithMaxDeltaElements(10);
var props = Replicator.Props(settings);
_replicator = Sys.ActorOf(props, "replicator");
_timeOut = Dilated(TimeSpan.FromSeconds(3.0));
_writeTwo = new WriteTo(2, _timeOut);
_writeMajority = new WriteMajority(_timeOut);
_writeAll = new WriteAll(_timeOut);
_readTwo = new ReadFrom(2, _timeOut);
_readMajority = new ReadMajority(_timeOut);
_readAll = new ReadAll(_timeOut);
}
public ReplicatorSpecs(ITestOutputHelper helper) : base(SpecConfig, helper)
{
_sys1 = Sys;
_sys3 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
_sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
_timeOut = Dilated(TimeSpan.FromSeconds(3.0));
_writeTwo = new WriteTo(2, _timeOut);
_writeMajority = new WriteMajority(_timeOut);
_writeAll = new WriteAll(_timeOut);
_readTwo = new ReadFrom(2, _timeOut);
_readMajority = new ReadMajority(_timeOut);
_readAll = new ReadAll(_timeOut);
var settings = ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1.0))
.WithMaxDeltaElements(10);
var props = Replicator.Props(settings);
_replicator1 = _sys1.ActorOf(props, "replicator");
_replicator2 = _sys2.ActorOf(props, "replicator");
_replicator3 = _sys3.ActorOf(props, "replicator");
}
public void Insert_from_5_nodes_should_write_to_majority_during_3_and_2_partition_and_read_from_majority_after_partition()
{
var key = new ORSetKey <int>("D");
var readMajority = new ReadMajority(_timeout);
var writeMajority = new WriteMajority(_timeout);
RunOn(() =>
{
SleepBeforePartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
SleepDuringPartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
EnterBarrier("partition-healed-4");
}, Controller);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1))));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
RunOn(() =>
{
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
}, N1, N4, N5);
RunOn(() =>
{
// without delays all could teoretically have been written before the blackhole
if (_delayMillis != 0)
{
failureWriteAcks.ShouldNotBe(new Replicator.IUpdateFailure[0]);
}
}, N2, N3);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
EnterBarrier("partition-healed-4");
// on the 2 node side, read from majority of nodes is enough to read all writes
RunOn(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}, N2, N3);
// but on the 3 node side, read from majority doesn't mean that we are guaranteed to see
// the writes from the other side, yet
// eventually all nodes will have the data
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}));
}, _nodes.ToArray());
EnterBarrier("after-test-4");
}
