public void Cluster_CRDT_should_replicate_values_to_new_node() { Join(_second, _first); RunOn(() => Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(2)); })), _first, _second); EnterBarrier("2-nodes"); RunOn(() => { var changedProbe = CreateTestProbe(); _replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref)); // "A" should be replicated via gossip to the new node Within(TimeSpan.FromSeconds(5), () => AwaitAssert(() => { //TODO: received message is NotFound(A) instead of GetSuccess // for some reason result is returned before CRDT gets replicated _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyA)).Get(KeyA); c.Value.ShouldBe(6); })); var c2 = changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, KeyA)).Get(KeyA); c2.Value.ShouldBe(6); }, _second); EnterBarrierAfterTestStep(); }
public PerformanceMonitor(TimeSpan updateInterval) { // schedule periodical updates this.cancelUpdates = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(updateInterval, updateInterval, Self, CollectMetrics.Instance, ActorRefs.NoSender); var cluster = Cluster.Get(Context.System); // create and start performance counters for a local node this.cpuCounter = new PerformanceCounter("Processor", "% Processor Time", "_Total"); this.memCounter = new PerformanceCounter("Memory", "Available MBytes"); cpuCounter.NextValue(); memCounter.NextValue(); // assign to a distributed store of performance metrics var perfCounterKey = new LWWDictionaryKey <UniqueAddress, PerformanceMetrics>("perf-counters"); var replicator = DistributedData.DistributedData.Get(Context.System).Replicator; replicator.Tell(Dsl.Subscribe(perfCounterKey, Self)); Receive <Replicator.Changed>(changed => { lastResult = lastResult.Merge(changed.Get(perfCounterKey)); }); Receive <CollectMetrics>(_ => { var cpuUsage = cpuCounter.NextValue(); var memUsage = memCounter.NextValue(); lastResult = lastResult.SetItem(cluster, cluster.SelfUniqueAddress, new PerformanceMetrics(cpuUsage, memUsage)); replicator.Tell(Dsl.Update(perfCounterKey, lastResult, WriteLocal.Instance, null, map => lastResult.Merge(map))); }); }
private async Task ReplicatorDuplicatePublish() { var p1 = CreateTestProbe(_sys1); var p2 = CreateTestProbe(_sys2); var p3 = CreateTestProbe(_sys3); var probes = new[] { p1, p2, p3 }; // subscribe to updates on all 3 nodes _replicator1.Tell(Dsl.Subscribe(_keyI, p1.Ref)); _replicator2.Tell(Dsl.Subscribe(_keyI, p2.Ref)); _replicator3.Tell(Dsl.Subscribe(_keyI, p3.Ref)); // update item on 2 _replicator2.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))); Sys.Log.Info("Pushed change from sys2 for I"); // wait for write to replicate to all 3 nodes Within(TimeSpan.FromSeconds(5), () => { foreach (var p in probes) { p.ExpectMsg <Changed>(c => c.Get(_keyI).Elements.ShouldBe(ImmutableHashSet.Create("a"))); } }); // create duplicate write on node 1 Sys.Log.Info("Pushing change from sys1 for I"); _replicator1.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))); // no probe should receive an update p2.ExpectNoMsg(TimeSpan.FromSeconds(1)); }
private void UpdateORDictionaryNode2And1() { var changedProbe = CreateTestProbe(_sys2); // subscribe to updates for KeyH, then update it with a replication factor of two _replicator2.Tell(Dsl.Subscribe(_keyH, changedProbe.Ref)); _replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "a", Flag.False))); // receive local update changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.False), })).ShouldBeTrue(); // push update from node 1 _replicator1.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys1), "a", Flag.True))); // expect replication of update on node 2 changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True) })).ShouldBeTrue(); // add new value to dictionary from node 2 _replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "b", Flag.True))); changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True), new KeyValuePair <string, Flag>("b", Flag.True) })).ShouldBeTrue(); }
public void Cluster_CRDT_should_avoid_duplicate_change_events_for_same_data() { var changedProbe = CreateTestProbe(); _replicator.Tell(Dsl.Subscribe(KeyI, changedProbe.Ref)); EnterBarrier("subscribed-I"); RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))), _second); Within(TimeSpan.FromSeconds(5), () => { var changed = changedProbe.ExpectMsg <Changed>(c => c.Get(KeyI).Elements.ShouldBe(ImmutableHashSet.Create("a"))); var keyIData = changed.Get(KeyI); Sys.Log.Debug("DEBUG: Received Changed {0}", changed); }); EnterBarrier("update-I"); RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))), _first); changedProbe.ExpectNoMsg(TimeSpan.FromSeconds(1)); EnterBarrierAfterTestStep(); }
public void Durable_CRDT_should_be_durable_after_gossip_update() { var r = NewReplicator(); RunOn(() => { Log.Debug("sending message with sender: {}", ActorCell.GetCurrentSelfOrNoSender()); r.Tell(Dsl.Update(_keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(_cluster, Myself.Name))); ExpectMsg(new UpdateSuccess(_keyC, null)); }, _first); RunOn(() => { r.Tell(Dsl.Subscribe(_keyC, TestActor)); ExpectMsg <Changed>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name)); // must do one more roundtrip to be sure that it keyB is stored, since Changed might have // been sent out before storage TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 3)); ExpectMsg(new UpdateSuccess(_keyA, null)); Watch(r); Sys.Stop(r); ExpectTerminated(r); var r2 = default(IActorRef); AwaitAssert(() => r2 = NewReplicator()); // try until name is free AwaitAssert(() => { r2.Tell(Dsl.GetKeyIds); ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty); }); r2.Tell(Dsl.Get(_keyC, ReadLocal.Instance)); ExpectMsg <GetSuccess>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name)); }, _second); EnterBarrierAfterTestStep(); }
public void Cluster_CRDT_should_check_that_remote_update_and_local_update_both_cause_a_change_event_to_emit_with_the_merged_data() { var changedProbe = CreateTestProbe(); RunOn(() => { _replicator.Tell(Dsl.Subscribe(KeyH, changedProbe.Ref)); _replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.False))); ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.False), })).ShouldBeTrue(); }, _second); EnterBarrier("update-h1"); RunOn(() => { _replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.True))); }, _first); RunOn(() => { changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True) })).ShouldBeTrue(); _replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "b", Flag.True))); changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[] { new KeyValuePair <string, Flag>("a", Flag.True), new KeyValuePair <string, Flag>("b", Flag.True) })).ShouldBeTrue(); }, _second); EnterBarrierAfterTestStep(); }
public void Durable_CRDT_should_be_durable_after_gossip_update() { var r = NewReplicator(Sys); RunOn(() => { r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(cluster, Myself.Name))); ExpectMsg(new UpdateSuccess(keyC, null)); }, first); RunOn(() => { r.Tell(Dsl.Subscribe(keyC, TestActor)); ExpectMsg <Changed>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name)); // must do one more roundtrip to be sure that it keyB is stored, since Changed might have // been sent out before storage r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster))); ExpectMsg(new UpdateSuccess(keyA, null)); Watch(r); Sys.Stop(r); ExpectTerminated(r); var r2 = default(IActorRef); AwaitAssert(() => r2 = NewReplicator(Sys)); AwaitAssert(() => { r2.Tell(Dsl.GetKeyIds); ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty); }); r2.Tell(Dsl.Get(keyC, ReadLocal.Instance)); ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name)); }, second); EnterBarrierAfterTestStep(); }
public DDataShardCoordinator(string typeName, ClusterShardingSettings settings, IShardAllocationStrategy allocationStrategy, IActorRef replicator, int majorityMinCap, bool rememberEntities) { _replicator = replicator; _rememberEntities = rememberEntities; Settings = settings; AllocationStrategy = allocationStrategy; Log = Context.GetLogger(); Cluster = Cluster.Get(Context.System); CurrentState = PersistentShardCoordinator.State.Empty.WithRememberEntities(settings.RememberEntities); RemovalMargin = Cluster.DowningProvider.DownRemovalMargin; MinMembers = string.IsNullOrEmpty(settings.Role) ? Cluster.Settings.MinNrOfMembers : (Cluster.Settings.MinNrOfMembersOfRole.TryGetValue(settings.Role, out var min) ? min : Cluster.Settings.MinNrOfMembers); RebalanceTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(Settings.TunningParameters.RebalanceInterval, Settings.TunningParameters.RebalanceInterval, Self, RebalanceTick.Instance, Self); _readConsistency = new ReadMajority(settings.TunningParameters.WaitingForStateTimeout, majorityMinCap); _writeConsistency = new WriteMajority(settings.TunningParameters.UpdatingStateTimeout, majorityMinCap); _coordinatorStateKey = new LWWRegisterKey <PersistentShardCoordinator.State>(typeName + "CoordinatorState"); _allShardsKey = new GSetKey <string>($"shard-{typeName}-all"); _allKeys = rememberEntities ? ImmutableHashSet.CreateRange(new IKey <IReplicatedData>[] { _coordinatorStateKey, _allShardsKey }) : ImmutableHashSet.Create <IKey <IReplicatedData> >(_coordinatorStateKey); if (rememberEntities) { replicator.Tell(Dsl.Subscribe(_allShardsKey, Self)); } Cluster.Subscribe(Self, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents, typeof(ClusterEvent.ClusterShuttingDown)); // get state from ddata replicator, repeat until GetSuccess GetCoordinatorState(); GetAllShards(); Context.Become(WaitingForState(_allKeys)); }
public void Cluster_CRDT_should_be_replicated_after_successful_update() { var changedProbe = CreateTestProbe(); RunOn(() => { _replicator.Tell(Dsl.Subscribe(KeyC, changedProbe.Ref)); }, _first, _second); RunOn(() => { _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30))); ExpectMsg(new UpdateSuccess(KeyC, null)); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30); _replicator.Tell(Dsl.Update(KeyY, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30))); ExpectMsg(new UpdateSuccess(KeyY, null)); _replicator.Tell(Dsl.Update(KeyZ, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 30))); ExpectMsg(new UpdateSuccess(KeyZ, null)); }, _first); EnterBarrier("update-c30"); RunOn(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c30 = ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyC)).Get(KeyC); c30.Value.ShouldBe(30); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30); // replicate with gossip after WriteLocal _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyC, null)); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31); _replicator.Tell(Dsl.Delete(KeyY, WriteLocal.Instance, 777)); ExpectMsg(new DeleteSuccess(KeyY, 777)); _replicator.Tell(Dsl.Get(KeyZ, _readMajority)); ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyZ)).Get(KeyZ).Value.ShouldBe(30); }, _second); EnterBarrier("update-c31"); RunOn(() => { // KeyC and deleted KeyY should be replicated via gossip to the other node Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC); c.Value.ShouldBe(31); _replicator.Tell(Dsl.Get(KeyY, ReadLocal.Instance)); ExpectMsg(new DataDeleted(KeyY)); }); }); changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31); }, _first); EnterBarrier("verified-c31"); // and also for concurrent updates RunOn(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c31 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC); c31.Value.ShouldBe(31); _replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1))); ExpectMsg(new UpdateSuccess(KeyC, null)); Within(TimeSpan.FromSeconds(5), () => { AwaitAssert(() => { _replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance)); var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC); c.Value.ShouldBe(33); }); }); }, _first, _second); EnterBarrierAfterTestStep(); }
public void Cluster_CRDT_should_work_in_single_node_cluster() { Join(_first, _first); RunOn(() => { Within(TimeSpan.FromSeconds(5.0), () => { _replicator.Tell(Dsl.GetReplicaCount); ExpectMsg(new ReplicaCount(1)); }); var changedProbe = CreateTestProbe(); _replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref)); _replicator.Tell(Dsl.Subscribe(KeyX, changedProbe.Ref)); _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); ExpectMsg(new NotFound(KeyA, null)); var c3 = GCounter.Empty.Increment(_cluster, 3); var update = Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 3)); _replicator.Tell(update); ExpectMsg(new UpdateSuccess(KeyA, null)); _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); ExpectMsg(new GetSuccess(KeyA, null, c3)); changedProbe.ExpectMsg(new Changed(KeyA, c3)); var changedProbe2 = CreateTestProbe(); _replicator.Tell(new Subscribe(KeyA, changedProbe2.Ref)); changedProbe2.ExpectMsg(new Changed(KeyA, c3)); var c4 = c3.Increment(_cluster); // too strong consistency level _replicator.Tell(Dsl.Update(KeyA, _writeTwo, x => x.Increment(_cluster))); ExpectMsg(new UpdateTimeout(KeyA, null), _timeOut.Add(TimeSpan.FromSeconds(1))); _replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance)); ExpectMsg(new GetSuccess(KeyA, null, c4)); changedProbe.ExpectMsg(new Changed(KeyA, c4)); var c5 = c4.Increment(_cluster); // too strong consistency level _replicator.Tell(Dsl.Update(KeyA, _writeMajority, x => x.Increment(_cluster))); ExpectMsg(new UpdateSuccess(KeyA, null)); _replicator.Tell(Dsl.Get(KeyA, _readMajority)); ExpectMsg(new GetSuccess(KeyA, null, c5)); changedProbe.ExpectMsg(new Changed(KeyA, c5)); var c6 = c5.Increment(_cluster); _replicator.Tell(Dsl.Update(KeyA, _writeAll, x => x.Increment(_cluster))); ExpectMsg(new UpdateSuccess(KeyA, null)); _replicator.Tell(Dsl.Get(KeyA, _readAll)); ExpectMsg(new GetSuccess(KeyA, null, c6)); changedProbe.ExpectMsg(new Changed(KeyA, c6)); var c9 = GCounter.Empty.Increment(_cluster, 9); _replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 9))); ExpectMsg(new UpdateSuccess(KeyX, null)); changedProbe.ExpectMsg(new Changed(KeyX, c9)); _replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance)); ExpectMsg(new DeleteSuccess(KeyX)); changedProbe.ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Get(KeyX, ReadLocal.Instance)); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Get(KeyX, _readAll)); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Update(KeyX, WriteLocal.Instance, x => x.Increment(_cluster))); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance)); ExpectMsg(new DataDeleted(KeyX)); _replicator.Tell(Dsl.GetKeyIds); ExpectMsg(new GetKeysIdsResult(ImmutableHashSet <string> .Empty.Add("A"))); }, _first); EnterBarrierAfterTestStep(); }