public void Cluster_CRDT_should_replicate_values_to_new_node()
{
Join(_second, _first);
RunOn(() =>
Within(TimeSpan.FromSeconds(10), () =>
AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(2));
})),
_first, _second);
EnterBarrier("2-nodes");
RunOn(() =>
{
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref));
// "A" should be replicated via gossip to the new node
Within(TimeSpan.FromSeconds(5), () =>
AwaitAssert(() =>
{
//TODO: received message is NotFound(A) instead of GetSuccess
// for some reason result is returned before CRDT gets replicated
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyA)).Get(KeyA);
c.Value.ShouldBe(6);
}));
var c2 = changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, KeyA)).Get(KeyA);
c2.Value.ShouldBe(6);
}, _second);
EnterBarrierAfterTestStep();
}
public PerformanceMonitor(TimeSpan updateInterval)
{
// schedule periodical updates
this.cancelUpdates = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(updateInterval, updateInterval, Self, CollectMetrics.Instance, ActorRefs.NoSender);
var cluster = Cluster.Get(Context.System);
// create and start performance counters for a local node
this.cpuCounter = new PerformanceCounter("Processor", "% Processor Time", "_Total");
this.memCounter = new PerformanceCounter("Memory", "Available MBytes");
cpuCounter.NextValue();
memCounter.NextValue();
// assign to a distributed store of performance metrics
var perfCounterKey = new LWWDictionaryKey <UniqueAddress, PerformanceMetrics>("perf-counters");
var replicator = DistributedData.DistributedData.Get(Context.System).Replicator;
replicator.Tell(Dsl.Subscribe(perfCounterKey, Self));
Receive <Replicator.Changed>(changed =>
{
lastResult = lastResult.Merge(changed.Get(perfCounterKey));
});
Receive <CollectMetrics>(_ =>
{
var cpuUsage = cpuCounter.NextValue();
var memUsage = memCounter.NextValue();
lastResult = lastResult.SetItem(cluster, cluster.SelfUniqueAddress, new PerformanceMetrics(cpuUsage, memUsage));
replicator.Tell(Dsl.Update(perfCounterKey, lastResult, WriteLocal.Instance, null, map => lastResult.Merge(map)));
});
}
private async Task ReplicatorDuplicatePublish()
{
var p1 = CreateTestProbe(_sys1);
var p2 = CreateTestProbe(_sys2);
var p3 = CreateTestProbe(_sys3);
var probes = new[] { p1, p2, p3 };
// subscribe to updates on all 3 nodes
_replicator1.Tell(Dsl.Subscribe(_keyI, p1.Ref));
_replicator2.Tell(Dsl.Subscribe(_keyI, p2.Ref));
_replicator3.Tell(Dsl.Subscribe(_keyI, p3.Ref));
// update item on 2
_replicator2.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a")));
Sys.Log.Info("Pushed change from sys2 for I");
// wait for write to replicate to all 3 nodes
Within(TimeSpan.FromSeconds(5), () =>
{
foreach (var p in probes)
{
p.ExpectMsg <Changed>(c => c.Get(_keyI).Elements.ShouldBe(ImmutableHashSet.Create("a")));
}
});
// create duplicate write on node 1
Sys.Log.Info("Pushing change from sys1 for I");
_replicator1.Tell(Dsl.Update(_keyI, GSet <string> .Empty, _writeTwo, a => a.Add("a")));
// no probe should receive an update
p2.ExpectNoMsg(TimeSpan.FromSeconds(1));
}
private void UpdateORDictionaryNode2And1()
{
var changedProbe = CreateTestProbe(_sys2);
// subscribe to updates for KeyH, then update it with a replication factor of two
_replicator2.Tell(Dsl.Subscribe(_keyH, changedProbe.Ref));
_replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "a", Flag.False)));
// receive local update
changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.False),
})).ShouldBeTrue();
// push update from node 1
_replicator1.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys1), "a", Flag.True)));
// expect replication of update on node 2
changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True)
})).ShouldBeTrue();
// add new value to dictionary from node 2
_replicator2.Tell(Dsl.Update(_keyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(Cluster.Cluster.Get(_sys2), "b", Flag.True)));
changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, _keyH)).Get(_keyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True),
new KeyValuePair <string, Flag>("b", Flag.True)
})).ShouldBeTrue();
}
public void Cluster_CRDT_should_avoid_duplicate_change_events_for_same_data()
{
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyI, changedProbe.Ref));
EnterBarrier("subscribed-I");
RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))),
_second);
Within(TimeSpan.FromSeconds(5), () =>
{
var changed = changedProbe.ExpectMsg <Changed>(c =>
c.Get(KeyI).Elements.ShouldBe(ImmutableHashSet.Create("a")));
var keyIData = changed.Get(KeyI);
Sys.Log.Debug("DEBUG: Received Changed {0}", changed);
});
EnterBarrier("update-I");
RunOn(() => _replicator.Tell(Dsl.Update(KeyI, GSet <string> .Empty, _writeTwo, a => a.Add("a"))),
_first);
changedProbe.ExpectNoMsg(TimeSpan.FromSeconds(1));
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_be_durable_after_gossip_update()
{
var r = NewReplicator();
RunOn(() =>
{
Log.Debug("sending message with sender: {}", ActorCell.GetCurrentSelfOrNoSender());
r.Tell(Dsl.Update(_keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(_cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(_keyC, null));
}, _first);
RunOn(() =>
{
r.Tell(Dsl.Subscribe(_keyC, TestActor));
ExpectMsg <Changed>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name));
// must do one more roundtrip to be sure that it keyB is stored, since Changed might have
// been sent out before storage
TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 3));
ExpectMsg(new UpdateSuccess(_keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator()); // try until name is free
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(_keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name));
}, _second);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_check_that_remote_update_and_local_update_both_cause_a_change_event_to_emit_with_the_merged_data()
{
var changedProbe = CreateTestProbe();
RunOn(() =>
{
_replicator.Tell(Dsl.Subscribe(KeyH, changedProbe.Ref));
_replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.False)));
ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.False),
})).ShouldBeTrue();
}, _second);
EnterBarrier("update-h1");
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "a", Flag.True)));
}, _first);
RunOn(() =>
{
changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True)
})).ShouldBeTrue();
_replicator.Tell(Dsl.Update(KeyH, ORDictionary <string, Flag> .Empty, _writeTwo, x => x.SetItem(_cluster, "b", Flag.True)));
changedProbe.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyH)).Get(KeyH).Entries.SequenceEqual(ImmutableDictionary.CreateRange(new[]
{
new KeyValuePair <string, Flag>("a", Flag.True),
new KeyValuePair <string, Flag>("b", Flag.True)
})).ShouldBeTrue();
}, _second);
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_be_durable_after_gossip_update()
{
var r = NewReplicator(Sys);
RunOn(() =>
{
r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(keyC, null));
}, first);
RunOn(() =>
{
r.Tell(Dsl.Subscribe(keyC, TestActor));
ExpectMsg <Changed>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name));
// must do one more roundtrip to be sure that it keyB is stored, since Changed might have
// been sent out before storage
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator(Sys));
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name));
}, second);
EnterBarrierAfterTestStep();
}
public DDataShardCoordinator(string typeName, ClusterShardingSettings settings, IShardAllocationStrategy allocationStrategy, IActorRef replicator, int majorityMinCap, bool rememberEntities)
{
_replicator = replicator;
_rememberEntities = rememberEntities;
Settings = settings;
AllocationStrategy = allocationStrategy;
Log = Context.GetLogger();
Cluster = Cluster.Get(Context.System);
CurrentState = PersistentShardCoordinator.State.Empty.WithRememberEntities(settings.RememberEntities);
RemovalMargin = Cluster.DowningProvider.DownRemovalMargin;
MinMembers = string.IsNullOrEmpty(settings.Role)
? Cluster.Settings.MinNrOfMembers
: (Cluster.Settings.MinNrOfMembersOfRole.TryGetValue(settings.Role, out var min) ? min : Cluster.Settings.MinNrOfMembers);
RebalanceTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(Settings.TunningParameters.RebalanceInterval, Settings.TunningParameters.RebalanceInterval, Self, RebalanceTick.Instance, Self);
_readConsistency = new ReadMajority(settings.TunningParameters.WaitingForStateTimeout, majorityMinCap);
_writeConsistency = new WriteMajority(settings.TunningParameters.UpdatingStateTimeout, majorityMinCap);
_coordinatorStateKey = new LWWRegisterKey <PersistentShardCoordinator.State>(typeName + "CoordinatorState");
_allShardsKey = new GSetKey <string>($"shard-{typeName}-all");
_allKeys = rememberEntities
? ImmutableHashSet.CreateRange(new IKey <IReplicatedData>[] { _coordinatorStateKey, _allShardsKey })
: ImmutableHashSet.Create <IKey <IReplicatedData> >(_coordinatorStateKey);
if (rememberEntities)
{
replicator.Tell(Dsl.Subscribe(_allShardsKey, Self));
}
Cluster.Subscribe(Self, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents, typeof(ClusterEvent.ClusterShuttingDown));
// get state from ddata replicator, repeat until GetSuccess
GetCoordinatorState();
GetAllShards();
Context.Become(WaitingForState(_allKeys));
}
public void Cluster_CRDT_should_be_replicated_after_successful_update()
{
var changedProbe = CreateTestProbe();
RunOn(() =>
{
_replicator.Tell(Dsl.Subscribe(KeyC, changedProbe.Ref));
}, _first, _second);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
_replicator.Tell(Dsl.Update(KeyY, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyY, null));
_replicator.Tell(Dsl.Update(KeyZ, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyZ, null));
}, _first);
EnterBarrier("update-c30");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c30 = ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyC)).Get(KeyC);
c30.Value.ShouldBe(30);
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
// replicate with gossip after WriteLocal
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
_replicator.Tell(Dsl.Delete(KeyY, WriteLocal.Instance, 777));
ExpectMsg(new DeleteSuccess(KeyY, 777));
_replicator.Tell(Dsl.Get(KeyZ, _readMajority));
ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyZ)).Get(KeyZ).Value.ShouldBe(30);
}, _second);
EnterBarrier("update-c31");
RunOn(() =>
{
// KeyC and deleted KeyY should be replicated via gossip to the other node
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(31);
_replicator.Tell(Dsl.Get(KeyY, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyY));
});
});
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
}, _first);
EnterBarrier("verified-c31");
// and also for concurrent updates
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c31 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c31.Value.ShouldBe(31);
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(33);
});
});
}, _first, _second);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_work_in_single_node_cluster()
{
Join(_first, _first);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(5.0), () =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(1));
});
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref));
_replicator.Tell(Dsl.Subscribe(KeyX, changedProbe.Ref));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new NotFound(KeyA, null));
var c3 = GCounter.Empty.Increment(_cluster, 3);
var update = Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 3));
_replicator.Tell(update);
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c3));
changedProbe.ExpectMsg(new Changed(KeyA, c3));
var changedProbe2 = CreateTestProbe();
_replicator.Tell(new Subscribe(KeyA, changedProbe2.Ref));
changedProbe2.ExpectMsg(new Changed(KeyA, c3));
var c4 = c3.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeTwo, x => x.Increment(_cluster)));
ExpectMsg(new UpdateTimeout(KeyA, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c4));
changedProbe.ExpectMsg(new Changed(KeyA, c4));
var c5 = c4.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeMajority, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readMajority));
ExpectMsg(new GetSuccess(KeyA, null, c5));
changedProbe.ExpectMsg(new Changed(KeyA, c5));
var c6 = c5.Increment(_cluster);
_replicator.Tell(Dsl.Update(KeyA, _writeAll, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readAll));
ExpectMsg(new GetSuccess(KeyA, null, c6));
changedProbe.ExpectMsg(new Changed(KeyA, c6));
var c9 = GCounter.Empty.Increment(_cluster, 9);
_replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 9)));
ExpectMsg(new UpdateSuccess(KeyX, null));
changedProbe.ExpectMsg(new Changed(KeyX, c9));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DeleteSuccess(KeyX));
changedProbe.ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, _readAll));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Update(KeyX, WriteLocal.Instance, x => x.Increment(_cluster)));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.GetKeyIds);
ExpectMsg(new GetKeysIdsResult(ImmutableHashSet <string> .Empty.Add("A")));
}, _first);
EnterBarrierAfterTestStep();
}
