public void Insert_from_5_nodes_should_read_write_to_majority_when_all_nodes_connected() { var key = new ORSetKey <int>("B"); var readMajority = new ReadMajority(_timeout); var writeMajority = new WriteMajority(_timeout); RunOn(() => { var writeProbe = CreateTestProbe(); var writeAcks = MyData.Select(i => { SleepDelay(); _replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref); return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1)))); }).ToArray(); var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray(); var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray(); successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray()); successWriteAcks.Length.ShouldBe(MyData.Count()); failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]); (successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count()); EnterBarrier("data-written-2"); // read from majority of nodes, which is enough to retrieve all data var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref); var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.ShouldBe(_expectedData); }, _nodes.ToArray()); RunOn(() => EnterBarrier("data-written-2"), Controller); EnterBarrier("after-test-2"); }
private void HandleMessage(CreateEventMarket message) { try { var cluster = Cluster.Get(Context.System); var replicator = DistributedData.Get(Context.System).Replicator; var key = new ORSetKey <string>($"Event-{message.EventId}"); var writeConsistency = new WriteMajority(TimeSpan.FromSeconds(2)); replicator.Tell(Dsl.Update(key, ORSet <string> .Empty, writeConsistency, existing => existing.Add(cluster, message.Market))); var localEvent = replicator.Ask <IGetResponse>(Dsl.Get(key, ReadLocal.Instance)); Sender.Tell(localEvent.Result); } catch (Exception e) { _log.Error(e, "Unable to process message CreateEventMarket for Event {0} Market {1}", message.EventId, message.Market); Sender.Tell( $"Unable to process message CreateEventMarket for Event {message.EventId} Market {message.Market}"); } }
public ReplicatorSpec(ReplicatorSpecConfig config) : base(config) { _config = config; _cluster = Cluster.Cluster.Get(Sys); var settings = ReplicatorSettings.Create(Sys).WithGossipInterval(TimeSpan.FromSeconds(1.0)).WithMaxDeltaElements(10); var props = Replicator.Props(settings); _replicator = Sys.ActorOf(props, "replicator"); _timeOut = Dilated(TimeSpan.FromSeconds(2.0)); _writeTwo = new WriteTo(2, _timeOut); _writeMajority = new WriteMajority(_timeOut); _writeAll = new WriteAll(_timeOut); _readTwo = new ReadFrom(2, _timeOut); _readMajority = new ReadMajority(_timeOut); _readAll = new ReadAll(_timeOut); }
protected ReplicatorSpec(ReplicatorSpecConfig config) : base(config, typeof(ReplicatorSpec)) { _config = config; _first = config.First; _second = config.Second; _third = config.Third; _cluster = Akka.Cluster.Cluster.Get(Sys); var settings = ReplicatorSettings.Create(Sys) .WithGossipInterval(TimeSpan.FromSeconds(1.0)) .WithMaxDeltaElements(10); var props = Replicator.Props(settings); _replicator = Sys.ActorOf(props, "replicator"); _timeOut = Dilated(TimeSpan.FromSeconds(3.0)); _writeTwo = new WriteTo(2, _timeOut); _writeMajority = new WriteMajority(_timeOut); _writeAll = new WriteAll(_timeOut); _readTwo = new ReadFrom(2, _timeOut); _readMajority = new ReadMajority(_timeOut); _readAll = new ReadAll(_timeOut); }
public ReplicatorSpecs(ITestOutputHelper helper) : base(SpecConfig, helper) { _sys1 = Sys; _sys3 = ActorSystem.Create(Sys.Name, Sys.Settings.Config); _sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config); _timeOut = Dilated(TimeSpan.FromSeconds(3.0)); _writeTwo = new WriteTo(2, _timeOut); _writeMajority = new WriteMajority(_timeOut); _writeAll = new WriteAll(_timeOut); _readTwo = new ReadFrom(2, _timeOut); _readMajority = new ReadMajority(_timeOut); _readAll = new ReadAll(_timeOut); var settings = ReplicatorSettings.Create(Sys) .WithGossipInterval(TimeSpan.FromSeconds(1.0)) .WithMaxDeltaElements(10); var props = Replicator.Props(settings); _replicator1 = _sys1.ActorOf(props, "replicator"); _replicator2 = _sys2.ActorOf(props, "replicator"); _replicator3 = _sys3.ActorOf(props, "replicator"); }
public void Insert_from_5_nodes_should_write_to_majority_during_3_and_2_partition_and_read_from_majority_after_partition() { var key = new ORSetKey <int>("D"); var readMajority = new ReadMajority(_timeout); var writeMajority = new WriteMajority(_timeout); RunOn(() => { SleepBeforePartition(); foreach (var a in new List <RoleName> { N1, N4, N5 }) { foreach (var b in new List <RoleName> { N2, N3 }) { TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3)); } } SleepDuringPartition(); foreach (var a in new List <RoleName> { N1, N4, N5 }) { foreach (var b in new List <RoleName> { N2, N3 }) { TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3)); } } EnterBarrier("partition-healed-4"); }, Controller); RunOn(() => { var writeProbe = CreateTestProbe(); var writeAcks = MyData.Select(i => { SleepDelay(); _replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref); return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1)))); }).ToArray(); var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray(); var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray(); RunOn(() => { successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray()); successWriteAcks.Length.ShouldBe(MyData.Count()); failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]); }, N1, N4, N5); RunOn(() => { // without delays all could teoretically have been written before the blackhole if (_delayMillis != 0) { failureWriteAcks.ShouldNotBe(new Replicator.IUpdateFailure[0]); } }, N2, N3); (successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count()); EnterBarrier("partition-healed-4"); // on the 2 node side, read from majority of nodes is enough to read all writes RunOn(() => { var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref); var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.ShouldBe(_expectedData); }, N2, N3); // but on the 3 node side, read from majority doesn't mean that we are guaranteed to see // the writes from the other side, yet // eventually all nodes will have the data Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() => { var readProbe = CreateTestProbe(); _replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref); var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key); result.Elements.ShouldBe(_expectedData); })); }, _nodes.ToArray()); EnterBarrier("after-test-4"); }