public void Insert_from_5_nodes_should_read_write_to_majority_when_all_nodes_connected()
{
var key = new ORSetKey <int>("B");
var readMajority = new ReadMajority(_timeout);
var writeMajority = new WriteMajority(_timeout);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1))));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
EnterBarrier("data-written-2");
// read from majority of nodes, which is enough to retrieve all data
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}, _nodes.ToArray());
RunOn(() => EnterBarrier("data-written-2"), Controller);
EnterBarrier("after-test-2");
}
//The way our data works, I opted to just removed all the key data, but leave the key intact as sometimes i need to use the key again in rare instances, but once deleted, ddata doesn't allow this.
private void HandleMessage(CloseEvent message)
{
try
{
var replicator = DistributedData.Get(Context.System).Replicator;
var cluster = Cluster.Get(Context.System);
var key = new ORSetKey <string>($"Event-{message.EventId}");
var writeConsistency = WriteLocal.Instance;
replicator.Tell(Dsl.Update(key, ORSet <string> .Empty, writeConsistency, $"Event-{message.EventId}",
existing =>
{
return(existing.Clear(cluster));
}));
var finalResult =
replicator.Ask <IGetResponse>(Dsl.Get(key, ReadLocal.Instance));
Sender.Tell(finalResult.Result);
}
catch (DataDeletedException e)
{
Sender.Tell($"Event {message.EventId} has been deleted");
}
catch (Exception e)
{
_log.Error(e, "Unable to process message CloseEvent for Event {0}", message.EventId);
Sender.Tell($"Unable to process message CloseEvent for Event {message.EventId}");
}
}
private void GetState()
{
for (int i = 0; i < NrOfKeys; i++)
{
Replicator.Tell(Dsl.Get(_stateKeys[i], _readConsistency, i));
}
}
public void Cluster_CRDT_should_replicate_values_to_new_node()
{
Join(_second, _first);
RunOn(() =>
Within(TimeSpan.FromSeconds(10), () =>
AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(2));
})),
_first, _second);
EnterBarrier("2-nodes");
RunOn(() =>
{
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref));
// "A" should be replicated via gossip to the new node
Within(TimeSpan.FromSeconds(5), () =>
AwaitAssert(() =>
{
//TODO: received message is NotFound(A) instead of GetSuccess
// for some reason result is returned before CRDT gets replicated
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyA)).Get(KeyA);
c.Value.ShouldBe(6);
}));
var c2 = changedProbe.ExpectMsg <Changed>(g => Equals(g.Key, KeyA)).Get(KeyA);
c2.Value.ShouldBe(6);
}, _second);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_converge_after_many_concurrent_updates()
{
Within(TimeSpan.FromSeconds(10), () =>
{
RunOn(() =>
{
var c = GCounter.Empty;
for (int i = 0; i < 100; i++)
{
c = c.Increment(_cluster, i);
_replicator.Tell(Dsl.Update(KeyF, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1)));
}
var results = ReceiveN(100);
results.All(x => x is UpdateSuccess).ShouldBeTrue();
}, _first, _second, _third);
EnterBarrier("100-updates-done");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyF, _readTwo));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyF)).Get(KeyF);
c.Value.ShouldBe(3 * 100);
}, _first, _second, _third);
EnterBarrierAfterTestStep();
});
}
private void HandleMessage(CreateEventMarket message)
{
try
{
var cluster = Cluster.Get(Context.System);
var replicator = DistributedData.Get(Context.System).Replicator;
var key = new ORSetKey <string>($"Event-{message.EventId}");
var writeConsistency = new WriteMajority(TimeSpan.FromSeconds(2));
replicator.Tell(Dsl.Update(key, ORSet <string> .Empty, writeConsistency,
existing => existing.Add(cluster, message.Market)));
var localEvent =
replicator.Ask <IGetResponse>(Dsl.Get(key, ReadLocal.Instance));
Sender.Tell(localEvent.Result);
}
catch (Exception e)
{
_log.Error(e, "Unable to process message CreateEventMarket for Event {0} Market {1}", message.EventId,
message.Market);
Sender.Tell(
$"Unable to process message CreateEventMarket for Event {message.EventId} Market {message.Market}");
}
}
public void Updates_from_same_node_should_be_possible_to_do_from_two_actors()
{
var updater1 = ActorOf(Props.Create <Updater>(), "updater1");
var updater2 = ActorOf(Props.Create <Updater>(), "updater2");
var b = ImmutableHashSet <string> .Empty.ToBuilder();
for (int i = 1; i <= 100; i++)
{
var m1 = "a" + 1;
var m2 = "b" + 1;
updater1.Tell(m1);
updater2.Tell(m2);
b.Add(m1);
b.Add(m2);
}
var expected = b.ToImmutable();
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(Updater.Key, ReadLocal.Instance));
var msg = ExpectMsg <GetSuccess>();
var elements = msg.Get(Updater.Key).Elements;
Assert.Equal(expected, elements);
});
}
private void AssertValue(IKey <IReplicatedData> key, object expected)
{
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance));
var g = ExpectMsg <GetSuccess>().Get(key);
object value;
switch (g)
{
case GCounter counter:
value = counter.Value;
break;
case PNCounter pnCounter:
value = pnCounter.Value;
break;
case GSet <string> set:
value = set.Elements;
break;
case ORSet <string> orSet:
value = orSet.Elements;
break;
default:
throw new ArgumentException("input doesn't match");
}
value.ShouldBe(expected);
}));
}
private void AssertValue(IKey <IReplicatedData> key, object expected)
{
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance));
var g = ExpectMsg <GetSuccess>().Get(key);
object value;
if (g is GCounter)
{
value = ((GCounter)g).Value;
}
else if (g is PNCounter)
{
value = ((PNCounter)g).Value;
}
else if (g is GSet <string> )
{
value = ((GSet <string>)g).Elements;
}
else if (g is ORSet <string> )
{
value = ((ORSet <string>)g).Elements;
}
else
{
throw new ArgumentException("input doesn't match");
}
value.ShouldBe(expected);
}));
}
private static async Task Main()
{
Console.WriteLine("Put the ddata mdb file in a folder called cluster-data in the application root folder. Press a key when done");
Console.Read();
var cfg = ConfigurationFactory.ParseString(File.ReadAllText("HOCON"))
.WithFallback(DistributedData.DefaultConfig());
var originalColour = Console.ForegroundColor;
var sys = ActorSystem.Create("test", cfg);
var dd = DistributedData.Get(sys);
int emptyKeyCount = 0;
var resp = await dd.Replicator.Ask <GetKeysIdsResult>(Dsl.GetKeyIds);
foreach (var resultKey in resp.Keys)
{
var key = new ORSetKey <string>($"{resultKey}");
var keyResp = await dd.Replicator.Ask <IGetResponse>(Dsl.Get(key));
Console.ForegroundColor = ConsoleColor.Green;
if (keyResp.Get(key).Elements.Count == 0)
{
emptyKeyCount++;
}
Console.WriteLine($"{key.Id}\t{string.Join<string>(",", keyResp.Get(key).Elements)}");
}
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"Finished loading {resp.Keys.Count} keys. There were {emptyKeyCount} empty keys");
Console.ForegroundColor = originalColour;
}
public void Insert_from_5_nodes_should_replicate_values_when_all_nodes_connected()
{
var key = new ORSetKey <int>("A");
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, WriteLocal.Instance, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(TimeSpan.FromSeconds(3)));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
// eventually all nodes will have the data
Within(TimeSpan.FromSeconds(15), () =>
{
AwaitAssert(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
});
});
}, _nodes.ToArray());
EnterBarrier("after-test-1");
}
public void Cluster_CRDT_should_work_in_2_node_cluster()
{
RunOn(() =>
{
// start with 20 on both nodes
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 20)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// add 1 on both nodes using WriteTwo
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// the total, after replication should be 42
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyB, _readTwo));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB);
c.Value.ShouldBe(42);
});
}, _first, _second);
EnterBarrier("update-42");
RunOn(() =>
{
// add 1 on both nodes using WriteAll
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeAll, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// the total, after replication should be 44
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyB, _readAll));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB);
c.Value.ShouldBe(44);
});
}, _first, _second);
EnterBarrier("update-44");
RunOn(() =>
{
// add 1 on both nodes using WriteMajority
_replicator.Tell(Dsl.Update(KeyB, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyB, null));
// the total, after replication should be 46
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyB, _readMajority));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyB)).Get(KeyB);
c.Value.ShouldBe(46);
});
}, _first, _second);
EnterBarrierAfterTestStep();
}
protected override void PreStart()
{
_pruningToken =
Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(
PruningConfiguration.PruningInterval,
PruningConfiguration.PruningInterval, Replicator.Replicator,
Dsl.Get(
new LWWDictionaryKey <string, DeduplicationState>(
ReplicatorKey), request: new PurgeStateQuery()), Self);
}
/// <summary>
/// On one of the nodes the data has been updated by the pruning, client can update anyway
/// </summary>
private void UpdateAfterPruning(int expectedValue)
{
_replicator.Tell(Dsl.Update(KeyA, GCounter.Empty, new WriteAll(_timeout), x => x.Increment(_cluster.SelfUniqueAddress, 1)));
ExpectMsg <Replicator.UpdateSuccess>(msg =>
{
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
var retrieved = ExpectMsg <Replicator.GetSuccess>().Get(KeyA);
retrieved.Value.ShouldBe(expectedValue);
});
}
public void Durable_CRDT_should_work_in_a_multi_node_cluster()
{
Join(second, first);
var r = NewReplicator(Sys);
Within(TimeSpan.FromSeconds(10), () =>
{
AwaitAssert(() =>
{
r.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(2));
});
});
EnterBarrier("both-initialized");
r.Tell(Dsl.Update(keyA, GCounter.Empty, writeTwo, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, writeTwo, c => c.Add(cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(keyC, null));
EnterBarrier("update-done-" + testStepCounter);
r.Tell(Dsl.Get(keyA, readTwo));
ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(2UL);
r.Tell(Dsl.Get(keyC, readTwo));
ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.CreateRange(new[] { first.Name, second.Name }));
EnterBarrier("values-verified-" + testStepCounter);
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator(Sys)); // try until name is free
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(keyA, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(2UL);
r2.Tell(Dsl.Get(keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.CreateRange(new[] { first.Name, second.Name }));
EnterBarrierAfterTestStep();
}
private void AssertDeleted(IKey <IReplicatedData> key)
{
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance));
ExpectMsg(new DataDeleted(key));
});
});
}
public void Cluster_CRDT_should_support_prefer_oldest_members()
{
// disable gossip and delta replication to only verify the write and read operations
var oldestReplicator = Sys.ActorOf(
Replicator.Props(
ReplicatorSettings.Create(Sys).WithPreferOldest(true).WithGossipInterval(TimeSpan.FromMinutes(1))),//.withDeltaCrdtEnabled(false)),
"oldestReplicator");
Within(TimeSpan.FromSeconds(5), () =>
{
var countProbe = CreateTestProbe();
AwaitAssert(() =>
{
oldestReplicator.Tell(GetReplicaCount.Instance, countProbe.Ref);
countProbe.ExpectMsg(new ReplicaCount(3));
});
});
EnterBarrier("oldest-replicator-started");
var probe = CreateTestProbe();
RunOn(() =>
{
oldestReplicator.Tell(
Dsl.Update(KeyK, new LWWRegister <string>(Cluster.SelfUniqueAddress, "0"), _writeTwo, a => a.WithValue(Cluster.SelfUniqueAddress, "1")),
probe.Ref);
probe.ExpectMsg(new UpdateSuccess(KeyK, null));
}, _second);
EnterBarrier("updated-1");
RunOn(() =>
{
// replicated to oldest
oldestReplicator.Tell(new Get(KeyK, ReadLocal.Instance), probe.Ref);
var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>);
((LWWRegister <string>)msg.Data).Value.Should().Be("1");
//probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1");
}, _first);
RunOn(() =>
{
// not replicated to third (not among the two oldest)
oldestReplicator.Tell(Dsl.Get(KeyK, ReadLocal.Instance), probe.Ref);
probe.ExpectMsg(new NotFound(KeyK, null));
// read from oldest
oldestReplicator.Tell(Dsl.Get(KeyK, _readTwo), probe.Ref);
var msg = probe.ExpectMsg <GetSuccess>(m => m.Data is LWWRegister <string>);
((LWWRegister <string>)msg.Data).Value.Should().Be("1");
//probe.ExpectMsg<GetSuccess[LWWRegister[String]]>.dataValue.value should === ("1");
}, _third);
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_work_in_a_single_node_cluster()
{
Join(_first, _first);
RunOn(() =>
{
var r = NewReplicator();
Within(TimeSpan.FromSeconds(10), () =>
{
AwaitAssert(() =>
{
r.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(1));
});
});
r.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
ExpectMsg(new NotFound(_keyA, null));
TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 1));
TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 2));
TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 3));
ExpectMsg(new UpdateSuccess(_keyA, null));
ExpectMsg(new UpdateSuccess(_keyA, null));
ExpectMsg(new UpdateSuccess(_keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator()); // try until name is free
// note that it will stash the commands until loading completed
r2.Tell(Dsl.Get(_keyA, ReadLocal.Instance));
var success = ExpectMsg <GetSuccess>().Get(_keyA);
success.TryGetValue("Id_1", out var value).Should().BeTrue();
value.TransactionId.Should().Be("Id_1");
value.Created.Should().Be(3);
Watch(r2);
Sys.Stop(r2);
ExpectTerminated(r2);
}, _first);
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_work_in_a_single_node_cluster()
{
Join(first, second);
RunOn(() =>
{
var r = NewReplicator(Sys);
Within(TimeSpan.FromSeconds(10), () =>
{
AwaitAssert(() =>
{
r.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(1));
});
});
r.Tell(Dsl.Get(keyA, ReadLocal.Instance));
ExpectMsg(new NotFound(keyA, null));
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
ExpectMsg(new UpdateSuccess(keyA, null));
ExpectMsg(new UpdateSuccess(keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator(Sys)); // try until name is free
// note that it will stash the commands until loading completed
r2.Tell(Dsl.Get(keyA, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyA).Value.ShouldBe(3UL);
Watch(r2);
Sys.Stop(r2);
ExpectTerminated(r2);
}, first);
EnterBarrierAfterTestStep();
}
public async Task <IEnumerable <MatchingActor> > GetMatchingActorsAsync(M message, IMessagePatternFactory <P> messagePatternFactory)
{
var replicator = DistributedData.Get(ActorSystem).Replicator;
GetKeysIdsResult keys = await replicator.Ask <GetKeysIdsResult>(Dsl.GetKeyIds);
var matchingActors = new List <MatchingActor>();
foreach (var node in keys.Keys)
{
var setKey = new LWWRegisterKey <ActorProps[]>(node);
var getResponse = await replicator.Ask <IGetResponse>(Dsl.Get(setKey, ReadLocal.Instance));
if (!getResponse.IsSuccessful)
{
throw new Exception($"cannot get message patterns for node {node}");
}
var actorPropsList = getResponse.Get(setKey).Value;
foreach (var actorProps in actorPropsList)
{
var path = actorProps.Path;
var matchingPatterns = actorProps.Patterns.Where(pattern => pattern.Match(message));
int matchingPatternsCount = matchingPatterns.Count();
if (matchingPatternsCount > 0)
{
var matchingPattern = matchingPatterns.First();
if (matchingPatternsCount > 1)
{
Logger.Warning("For actor {0}, found {1} patterns matching message {2}. Taking first one = {3}", path, matchingPatternsCount, message, matchingPattern);
}
matchingActors.Add(
new MatchingActor
{
Path = path,
IsSecondary = matchingPattern.IsSecondary,
MatchingScore = matchingPattern.Conjuncts.Length,
MistrustFactor = actorProps.MistrustFactor
});
}
}
}
return(matchingActors);
}
public void Durable_CRDT_should_be_durable_after_gossip_update()
{
var r = NewReplicator();
RunOn(() =>
{
Log.Debug("sending message with sender: {}", ActorCell.GetCurrentSelfOrNoSender());
r.Tell(Dsl.Update(_keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(_cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(_keyC, null));
}, _first);
RunOn(() =>
{
r.Tell(Dsl.Subscribe(_keyC, TestActor));
ExpectMsg <Changed>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name));
// must do one more roundtrip to be sure that it keyB is stored, since Changed might have
// been sent out before storage
TellUpdate(r, _keyA, WriteLocal.Instance, new PocoObject("Id_1", 3));
ExpectMsg(new UpdateSuccess(_keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator()); // try until name is free
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(_keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(_keyC).Elements.ShouldBe(ImmutableHashSet.Create(_first.Name));
}, _second);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_read_repair_happens_before_GetSuccess()
{
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyG, ORSet <string> .Empty, _writeTwo, x => x
.Add(_cluster, "a")
.Add(_cluster, "b")));
ExpectMsg <UpdateSuccess>();
}, _first);
EnterBarrier("a-b-added-to-G");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyG, _readAll));
ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyG)).Get(KeyG).Elements.SetEquals(new[] { "a", "b" });
_replicator.Tell(Dsl.Get(KeyG, ReadLocal.Instance));
ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyG)).Get(KeyG).Elements.SetEquals(new[] { "a", "b" });
}, _second);
EnterBarrierAfterTestStep();
}
public void Durable_CRDT_should_be_durable_after_gossip_update()
{
var r = NewReplicator(Sys);
RunOn(() =>
{
r.Tell(Dsl.Update(keyC, ORSet <string> .Empty, WriteLocal.Instance, c => c.Add(cluster, Myself.Name)));
ExpectMsg(new UpdateSuccess(keyC, null));
}, first);
RunOn(() =>
{
r.Tell(Dsl.Subscribe(keyC, TestActor));
ExpectMsg <Changed>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name));
// must do one more roundtrip to be sure that it keyB is stored, since Changed might have
// been sent out before storage
r.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
Watch(r);
Sys.Stop(r);
ExpectTerminated(r);
var r2 = default(IActorRef);
AwaitAssert(() => r2 = NewReplicator(Sys));
AwaitAssert(() =>
{
r2.Tell(Dsl.GetKeyIds);
ExpectMsg <GetKeysIdsResult>().Keys.ShouldNotBe(ImmutableHashSet <string> .Empty);
});
r2.Tell(Dsl.Get(keyC, ReadLocal.Instance));
ExpectMsg <GetSuccess>().Get(keyC).Elements.ShouldBe(ImmutableHashSet.Create(first.Name));
}, second);
EnterBarrierAfterTestStep();
}
public void Insert_from_5_nodes_should_write_to_majority_during_3_and_2_partition_and_read_from_majority_after_partition()
{
var key = new ORSetKey <int>("D");
var readMajority = new ReadMajority(_timeout);
var writeMajority = new WriteMajority(_timeout);
RunOn(() =>
{
SleepBeforePartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
SleepDuringPartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
EnterBarrier("partition-healed-4");
}, Controller);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, writeMajority, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(_timeout.Add(TimeSpan.FromSeconds(1))));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
RunOn(() =>
{
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
}, N1, N4, N5);
RunOn(() =>
{
// without delays all could teoretically have been written before the blackhole
if (_delayMillis != 0)
{
failureWriteAcks.ShouldNotBe(new Replicator.IUpdateFailure[0]);
}
}, N2, N3);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
EnterBarrier("partition-healed-4");
// on the 2 node side, read from majority of nodes is enough to read all writes
RunOn(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, readMajority), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}, N2, N3);
// but on the 3 node side, read from majority doesn't mean that we are guaranteed to see
// the writes from the other side, yet
// eventually all nodes will have the data
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}));
}, _nodes.ToArray());
EnterBarrier("after-test-4");
}
public void Cluster_CRDT_should_support_majority_quorum_write_and_read_with_3_nodes_with_1_unreachable()
{
Join(_third, _first);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(3));
}));
}, _first, _second, _third);
EnterBarrier("3-nodes");
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 50)));
ExpectMsg(new UpdateSuccess(KeyE, null));
}, _first, _second, _third);
EnterBarrier("write-initial-majority");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyE, _readMajority));
var c150 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c150.Value.ShouldBe(150);
}, _first, _second, _third);
EnterBarrier("read-initial-majority");
RunOn(() =>
{
TestConductor.Blackhole(_first, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
TestConductor.Blackhole(_second, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
}, _first);
EnterBarrier("blackhole-third");
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyE, null));
}, _second);
EnterBarrier("local-update-from-second");
RunOn(() =>
{
// ReadMajority should retrive the previous update from second, before applying the modification
var probe1 = CreateTestProbe();
var probe2 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe2.Ref);
probe2.ExpectMsg <GetSuccess>();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, data =>
{
probe1.Ref.Tell(data.Value);
return(data.Increment(_cluster, 1));
}), probe2.Ref);
// verify read your own writes, without waiting for the UpdateSuccess reply
// note that the order of the replies are not defined, and therefore we use separate probes
var probe3 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe3.Ref);
probe1.ExpectMsg(151);
probe2.ExpectMsg(new UpdateSuccess(KeyE, null));
var c152 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c152.Value.ShouldBe(152);
}, _first);
EnterBarrier("majority-update-from-first");
RunOn(() =>
{
var probe1 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe1.Ref);
probe1.ExpectMsg <GetSuccess>();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 153, x => x.Increment(_cluster, 1)), probe1.Ref);
// verify read your own writes, without waiting for the UpdateSuccess reply
// note that the order of the replies are not defined, and therefore we use separate probes
var probe2 = CreateTestProbe();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 154, x => x.Increment(_cluster, 1)), probe2.Ref);
var probe3 = CreateTestProbe();
_replicator.Tell(Dsl.Update(KeyE, GCounter.Empty, _writeMajority, 155, x => x.Increment(_cluster, 1)), probe3.Ref);
var probe5 = CreateTestProbe();
_replicator.Tell(Dsl.Get(KeyE, _readMajority), probe5.Ref);
probe1.ExpectMsg(new UpdateSuccess(KeyE, 153));
probe2.ExpectMsg(new UpdateSuccess(KeyE, 154));
probe3.ExpectMsg(new UpdateSuccess(KeyE, 155));
var c155 = probe5.ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c155.Value.ShouldBe(155);
}, _second);
EnterBarrier("majority-update-from-second");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyE2, _readAll, 998));
ExpectMsg(new GetFailure(KeyE2, 998), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Get(KeyE2, Dsl.ReadLocal));
ExpectMsg(new NotFound(KeyE2, null));
}, _first, _second);
EnterBarrier("read-all-fail-update");
RunOn(() =>
{
TestConductor.PassThrough(_first, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
TestConductor.PassThrough(_second, _third, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(5));
}, _first);
EnterBarrier("passThrough-third");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyE, _readMajority));
var c155 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyE)).Get(KeyE);
c155.Value.ShouldBe(155);
}, _third);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_converge_after_partition()
{
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 40)));
ExpectMsg(new UpdateSuccess(KeyD, null));
TestConductor.Blackhole(_first, _second, ThrottleTransportAdapter.Direction.Both)
.Wait(TimeSpan.FromSeconds(10));
}, _first);
EnterBarrier("blackhole-first-second");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyD, ReadLocal.Instance));
var c40 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyD)).Get(KeyD);
c40.Value.ShouldBe(40);
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty.Increment(_cluster, 1), _writeTwo, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateTimeout(KeyD, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Update(KeyD, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateTimeout(KeyD, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
}, _first, _second);
RunOn(() =>
{
//TODO: for some reason this RunOn never gets called
for (int i = 1; i <= 30; i++)
{
var n = i;
var keydn = new GCounterKey("D" + n);
_replicator.Tell(Dsl.Update(keydn, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, n)));
ExpectMsg(new UpdateSuccess(keydn, null));
}
}, _first);
EnterBarrier("updates-during-partion");
RunOn(() =>
{
TestConductor.PassThrough(_first, _second, ThrottleTransportAdapter.Direction.Both)
.Wait(TimeSpan.FromSeconds(5));
}, _first);
EnterBarrier("passThrough-first-second");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyD, _readTwo));
var c44 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyD)).Get(KeyD);
c44.Value.ShouldBe(44);
Within(TimeSpan.FromSeconds(10), () =>
AwaitAssert(() =>
{
for (int i = 1; i <= 30; i++)
{
var keydn = new GCounterKey("D" + i);
_replicator.Tell(Dsl.Get(keydn, ReadLocal.Instance));
ExpectMsg <GetSuccess>(g => Equals(g.Key, keydn)).Get(keydn).Value.ShouldBe(i);
}
}));
}, _first, _second);
EnterBarrierAfterTestStep();
}
public void Cluster_CRDT_should_be_replicated_after_successful_update()
{
var changedProbe = CreateTestProbe();
RunOn(() =>
{
_replicator.Tell(Dsl.Subscribe(KeyC, changedProbe.Ref));
}, _first, _second);
RunOn(() =>
{
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
_replicator.Tell(Dsl.Update(KeyY, GCounter.Empty, _writeTwo, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyY, null));
_replicator.Tell(Dsl.Update(KeyZ, GCounter.Empty, _writeMajority, x => x.Increment(_cluster, 30)));
ExpectMsg(new UpdateSuccess(KeyZ, null));
}, _first);
EnterBarrier("update-c30");
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c30 = ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyC)).Get(KeyC);
c30.Value.ShouldBe(30);
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(30);
// replicate with gossip after WriteLocal
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
_replicator.Tell(Dsl.Delete(KeyY, WriteLocal.Instance, 777));
ExpectMsg(new DeleteSuccess(KeyY, 777));
_replicator.Tell(Dsl.Get(KeyZ, _readMajority));
ExpectMsg <GetSuccess>(c => Equals(c.Key, KeyZ)).Get(KeyZ).Value.ShouldBe(30);
}, _second);
EnterBarrier("update-c31");
RunOn(() =>
{
// KeyC and deleted KeyY should be replicated via gossip to the other node
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(31);
_replicator.Tell(Dsl.Get(KeyY, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyY));
});
});
changedProbe.ExpectMsg <Changed>(c => Equals(c.Key, KeyC)).Get(KeyC).Value.ShouldBe(31);
}, _first);
EnterBarrier("verified-c31");
// and also for concurrent updates
RunOn(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c31 = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c31.Value.ShouldBe(31);
_replicator.Tell(Dsl.Update(KeyC, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 1)));
ExpectMsg(new UpdateSuccess(KeyC, null));
Within(TimeSpan.FromSeconds(5), () =>
{
AwaitAssert(() =>
{
_replicator.Tell(Dsl.Get(KeyC, ReadLocal.Instance));
var c = ExpectMsg <GetSuccess>(g => Equals(g.Key, KeyC)).Get(KeyC);
c.Value.ShouldBe(33);
});
});
}, _first, _second);
EnterBarrierAfterTestStep();
}
public void Insert_from_5_nodes_should_replicate_values_after_partition()
{
var key = new ORSetKey <int>("C");
RunOn(() =>
{
SleepBeforePartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
SleepDuringPartition();
foreach (var a in new List <RoleName> {
N1, N4, N5
})
{
foreach (var b in new List <RoleName> {
N2, N3
})
{
TestConductor.PassThrough(a, b, ThrottleTransportAdapter.Direction.Both).Wait(TimeSpan.FromSeconds(3));
}
}
EnterBarrier("partition-healed-3");
}, Controller);
RunOn(() =>
{
var writeProbe = CreateTestProbe();
var writeAcks = MyData.Select(i =>
{
SleepDelay();
_replicator.Tell(Dsl.Update(key, ORSet <int> .Empty, WriteLocal.Instance, i, x => x.Add(_cluster.SelfUniqueAddress, i)), writeProbe.Ref);
return(writeProbe.ReceiveOne(TimeSpan.FromSeconds(3)));
}).ToArray();
var successWriteAcks = writeAcks.OfType <Replicator.UpdateSuccess>().ToArray();
var failureWriteAcks = writeAcks.OfType <Replicator.IUpdateFailure>().ToArray();
successWriteAcks.Select(x => (int)x.Request).ShouldBe(MyData.ToArray());
successWriteAcks.Length.ShouldBe(MyData.Count());
failureWriteAcks.ShouldBe(new Replicator.IUpdateFailure[0]);
(successWriteAcks.Length + failureWriteAcks.Length).ShouldBe(MyData.Count());
EnterBarrier("partition-healed-3");
// eventually all nodes will have the data
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
var readProbe = CreateTestProbe();
_replicator.Tell(Dsl.Get(key, ReadLocal.Instance), readProbe.Ref);
var result = readProbe.ExpectMsg <Replicator.GetSuccess>(g => Equals(g.Key, key)).Get(key);
result.Elements.ShouldBe(_expectedData);
}));
}, _nodes.ToArray());
EnterBarrier("after-test-3");
}
public void Pruning_of_durable_CRDT_should_move_data_from_removed_node()
{
Join(first, first);
Join(second, first);
var sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
var cluster2 = Akka.Cluster.Cluster.Get(sys2);
var replicator2 = StartReplicator(sys2);
var probe2 = new TestProbe(sys2, new XunitAssertions());
cluster2.Join(Node(first).Address);
Within(TimeSpan.FromSeconds(5), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(4));
replicator2.Tell(Dsl.GetReplicaCount, probe2.Ref);
probe2.ExpectMsg(new ReplicaCount(4));
}));
replicator.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster)));
ExpectMsg(new UpdateSuccess(keyA, null));
replicator2.Tell(Dsl.Update(keyA, GCounter.Empty, WriteLocal.Instance, c => c.Increment(cluster2, 2)), probe2.Ref);
probe2.ExpectMsg(new UpdateSuccess(keyA, null));
EnterBarrier("updates-done");
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))));
var counter1 = ExpectMsg <GetSuccess>().Get(keyA);
counter1.Value.ShouldBe(10UL);
counter1.State.Count.ShouldBe(4);
}));
Within(TimeSpan.FromSeconds(10), () => AwaitAssert(() =>
{
replicator2.Tell(Dsl.Get(keyA, new ReadAll(TimeSpan.FromSeconds(1))), probe2.Ref);
var counter2 = probe2.ExpectMsg <GetSuccess>().Get(keyA);
counter2.Value.ShouldBe(10UL);
counter2.State.Count.ShouldBe(4);
}));
EnterBarrier("get1");
RunOn(() => cluster.Leave(cluster2.SelfAddress), first);
Within(TimeSpan.FromSeconds(15), () => AwaitAssert(() =>
{
replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(3));
}));
EnterBarrier("removed");
RunOn(() => sys2.Terminate().Wait(TimeSpan.FromSeconds(5)), first);
Within(TimeSpan.FromSeconds(15), () =>
{
var values = ImmutableHashSet <int> .Empty;
AwaitAssert(() =>
{
replicator.Tell(Dsl.Get(keyA, ReadLocal.Instance));
var counter3 = ExpectMsg <GetSuccess>().Get(keyA);
var value = counter3.Value;
values = values.Add((int)value);
value.ShouldBe(10UL);
counter3.State.Count.ShouldBe(3);
});
values.ShouldBe(ImmutableHashSet.Create(10));
});
EnterBarrier("prunned");
RunOn(() =>
{
var addr = cluster2.SelfAddress;
var sys3 = ActorSystem.Create(Sys.Name, ConfigurationFactory.ParseString(@"
").WithFallback(Sys.Settings.Config));
var cluster3 = Akka.Cluster.Cluster.Get(sys3);
var replicator3 = StartReplicator(sys3);
var probe3 = new TestProbe(sys3, new XunitAssertions());
cluster3.Join(Node(first).Address);
Within(TimeSpan.FromSeconds(10), () =>
{
var values = ImmutableHashSet <int> .Empty;
AwaitAssert(() =>
{
replicator3.Tell(Dsl.Get(keyA, ReadLocal.Instance), probe3.Ref);
var counter4 = probe3.ExpectMsg <GetSuccess>().Get(keyA);
var value = counter4.Value;
values.Add((int)value);
value.ShouldBe(10UL);
counter4.State.Count.ShouldBe(3);
});
values.ShouldBe(ImmutableHashSet.Create(10));
});
// after merging with others
replicator3.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault)));
var counter5 = ExpectMsg <GetSuccess>().Get(keyA);
counter5.Value.ShouldBe(10UL);
counter5.State.Count.ShouldBe(3);
}, first);
EnterBarrier("sys3-started");
replicator.Tell(Dsl.Get(keyA, new ReadAll(RemainingOrDefault)));
var counter6 = ExpectMsg <GetSuccess>().Get(keyA);
counter6.Value.ShouldBe(10UL);
counter6.State.Count.ShouldBe(3);
EnterBarrier("after-1");
}
public void Cluster_CRDT_should_work_in_single_node_cluster()
{
Join(_first, _first);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(5.0), () =>
{
_replicator.Tell(Dsl.GetReplicaCount);
ExpectMsg(new ReplicaCount(1));
});
var changedProbe = CreateTestProbe();
_replicator.Tell(Dsl.Subscribe(KeyA, changedProbe.Ref));
_replicator.Tell(Dsl.Subscribe(KeyX, changedProbe.Ref));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new NotFound(KeyA, null));
var c3 = GCounter.Empty.Increment(_cluster, 3);
var update = Dsl.Update(KeyA, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 3));
_replicator.Tell(update);
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c3));
changedProbe.ExpectMsg(new Changed(KeyA, c3));
var changedProbe2 = CreateTestProbe();
_replicator.Tell(new Subscribe(KeyA, changedProbe2.Ref));
changedProbe2.ExpectMsg(new Changed(KeyA, c3));
var c4 = c3.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeTwo, x => x.Increment(_cluster)));
ExpectMsg(new UpdateTimeout(KeyA, null), _timeOut.Add(TimeSpan.FromSeconds(1)));
_replicator.Tell(Dsl.Get(KeyA, ReadLocal.Instance));
ExpectMsg(new GetSuccess(KeyA, null, c4));
changedProbe.ExpectMsg(new Changed(KeyA, c4));
var c5 = c4.Increment(_cluster);
// too strong consistency level
_replicator.Tell(Dsl.Update(KeyA, _writeMajority, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readMajority));
ExpectMsg(new GetSuccess(KeyA, null, c5));
changedProbe.ExpectMsg(new Changed(KeyA, c5));
var c6 = c5.Increment(_cluster);
_replicator.Tell(Dsl.Update(KeyA, _writeAll, x => x.Increment(_cluster)));
ExpectMsg(new UpdateSuccess(KeyA, null));
_replicator.Tell(Dsl.Get(KeyA, _readAll));
ExpectMsg(new GetSuccess(KeyA, null, c6));
changedProbe.ExpectMsg(new Changed(KeyA, c6));
var c9 = GCounter.Empty.Increment(_cluster, 9);
_replicator.Tell(Dsl.Update(KeyX, GCounter.Empty, WriteLocal.Instance, x => x.Increment(_cluster, 9)));
ExpectMsg(new UpdateSuccess(KeyX, null));
changedProbe.ExpectMsg(new Changed(KeyX, c9));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DeleteSuccess(KeyX));
changedProbe.ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, ReadLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Get(KeyX, _readAll));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Update(KeyX, WriteLocal.Instance, x => x.Increment(_cluster)));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.Delete(KeyX, WriteLocal.Instance));
ExpectMsg(new DataDeleted(KeyX));
_replicator.Tell(Dsl.GetKeyIds);
ExpectMsg(new GetKeysIdsResult(ImmutableHashSet <string> .Empty.Add("A")));
}, _first);
EnterBarrierAfterTestStep();
}