private void Crash(params RoleName[] roles)
{
RunOn(() =>
{
foreach (var roleName in roles)
{
Log.Info("Shutdown [{0}]", GetAddress(roleName));
TestConductor.Exit(roleName, 0).Wait(TimeSpan.FromSeconds(10));
}
}, _config.Controller);
}
public void Cluster_sharding_with_remember_entities_should_start_remembered_entities_when_coordinator_fail_over()
{
Within(TimeSpan.FromSeconds(30), () =>
{
Join(_config.Second, _config.Second);
RunOn(() =>
{
StartSharding(Sys, TestActor);
_region.Value.Tell(1);
ExpectMsg <Started>();
}, _config.Second);
EnterBarrier("second-started");
Join(_config.Third, _config.Second);
RunOn(() =>
{
StartSharding(Sys, TestActor);
}, _config.Third);
RunOn(() =>
{
Within(Remaining, () =>
{
AwaitAssert(() =>
{
Cluster.State.Members.Count.Should().Be(2);
Cluster.State.Members.Should().OnlyContain(i => i.Status == MemberStatus.Up);
});
});
}, _config.Second, _config.Third);
EnterBarrier("all-up");
RunOn(() =>
{
if (IsDDataMode)
{
// Entity 1 in region of first node was started when there was only one node
// and then the remembering state will be replicated to second node by the
// gossip. So we must give that a chance to replicate before shutting down second.
Thread.Sleep(5000);
}
TestConductor.Exit(_config.Second, 0).Wait();
}, _config.First);
EnterBarrier("crash-second");
RunOn(() =>
{
ExpectMsg <Started>(Remaining);
}, _config.Third);
EnterBarrier("after-2");
});
}
private void Crash(params RoleName[] roles)
{
RunOn(() =>
{
foreach (var roleName in roles)
{
Log.Info("Shutdown [{0}]", roleName);
TestConductor.Exit(roleName, 0).Wait(TimeSpan.FromSeconds(10));
}
}, GetRole("controller") /*controller*/);
}
public void AMemberMustDetectFailureEvenThoughNoHeartbeatsHaveBeenReceived()
{
var firstAddress = GetAddress(_config.First);
var secondAddress = GetAddress(_config.Second);
AwaitClusterUp(_config.First);
RunOn(() =>
AwaitAssert(() =>
{
Cluster.SendCurrentClusterState(TestActor);
Assert.True(
ExpectMsg <ClusterEvent.CurrentClusterState>(TimeSpan.FromMilliseconds(50))
.Members.Select(m => m.Address)
.Contains(secondAddress));
}, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(50))
, _config.First);
RunOn(() =>
{
Cluster.Join(GetAddress(_config.First));
AwaitAssert(() =>
{
Cluster.SendCurrentClusterState(TestActor);
Assert.True(
ExpectMsg <ClusterEvent.CurrentClusterState>(TimeSpan.FromMilliseconds(50))
.Members.Select(m => m.Address)
.Contains(firstAddress));
}, TimeSpan.FromSeconds(20), TimeSpan.FromMilliseconds(50));
}, _config.Second);
//TODO: Seem to be able to pass barriers once other node fails?
EnterBarrier("second-joined");
//TODO: Finish!
return;
// It is likely that second has not started heartbeating to first yet,
// and when it does the messages doesn't go through and the first extra heartbeat is triggered.
// If the first heartbeat arrives, it will detect the failure anyway but not really exercise the
// part that we are trying to test here.
RunOn(
() =>
TestConductor.Blackhole(_config.First, _config.Second, ThrottleTransportAdapter.Direction.Both)
.Wait(), _config.Controller);
RunOn(
() => AwaitCondition(
() => !Cluster.FailureDetector.IsAvailable(GetAddress(_config.First))
, TimeSpan.FromSeconds(15))
, _config.Second);
EnterBarrier("after-1");
}
public void Replicated_ORSet_must_converge()
{
var probe = CreateTestProbe();
RunOn(() =>
{
var endpoint = this.CreateEndpoint(nodeA.Name, ImmutableHashSet <ReplicationConnection> .Empty.Add(Node(nodeB).Address.ToReplicationConnection()));
var service = new TestORSetService(Sys, "A", endpoint.Logs[this.LogName()], probe.Ref);
service.Add("x", 1);
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1));
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1, 2));
// network partition
TestConductor.Blackhole(nodeA, nodeB, ThrottleTransportAdapter.Direction.Both).Wait();
EnterBarrier("broken");
// this is concurrent to service.remove("x", 1) on node B
service.Add("x", 1);
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1, 2));
EnterBarrier("repair");
TestConductor.PassThrough(nodeA, nodeB, ThrottleTransportAdapter.Direction.Both).Wait();
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1, 2));
service.Remove("x", 2);
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1));
}, nodeA);
RunOn(() =>
{
var endpoint = this.CreateEndpoint(nodeA.Name, ImmutableHashSet <ReplicationConnection> .Empty.Add(Node(nodeA).Address.ToReplicationConnection()));
var service = new TestORSetService(Sys, "B", endpoint.Logs[this.LogName()], probe.Ref);
service.GetValue("x").Result.Should().BeEquivalentTo(1);
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1));
service.Add("x", 2);
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1, 2));
EnterBarrier("broken");
// this is concurrent to service.add("x", 1) on node A
service.Remove("x", 1);
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(2));
EnterBarrier("repair");
// add has precedence over (concurrent) remove
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1, 2));
probe.ExpectMsg <ImmutableHashSet <int> >(x => x.Should().BeEquivalentTo(1));
}, nodeB);
}
private void Crash(params RoleName[] roles)
{
RunOn(() =>
{
Queue.Tell(PointToPointChannel.Reset.Instance);
ExpectMsg <PointToPointChannel.ResetOk>();
foreach (var role in roles)
{
Log.Info("Shutdown [{0}]", GetAddress(role));
TestConductor.Exit(role, 0).Wait();
}
}, _controller);
}
private void RemoteNodeDeathWatch_must_receive_Terminated_when_watched_node_crash()
{
RunOn(() =>
{
var watcher = Sys.ActorOf(Props.Create(() => new ProbeActor(TestActor)), "watcher6");
var watcher2 = Sys.ActorOf(Props.Create(() => new ProbeActor(Sys.DeadLetters)));
EnterBarrier("actors-started-6");
var subject = _identify(_config.Second, "subject6");
watcher.Tell(new WatchIt(subject));
ExpectMsg <RemoteNodeDeathWatchMultiNetSpec.Ack>(TimeSpan.FromSeconds(1));
watcher2.Tell(new WatchIt(subject));
ExpectMsg <RemoteNodeDeathWatchMultiNetSpec.Ack>(TimeSpan.FromSeconds(1));
subject.Tell("hello6");
// testing with this watch/unwatch of watcher2 to make sure that the unwatch doesn't
// remove the first watch
watcher2.Tell(new UnwatchIt(subject));
ExpectMsg <RemoteNodeDeathWatchMultiNetSpec.Ack>(TimeSpan.FromSeconds(1));
EnterBarrier("watch-established-6");
Sleep();
Log.Info("exit second");
TestConductor.Exit(_config.Second, 0).Wait();
ExpectMsg <WrappedTerminated>(TimeSpan.FromSeconds(15)).T.ActorRef.ShouldBe(subject);
// verify that things are cleaned up, and heartbeating is stopped
AssertCleanup();
ExpectNoMsg(TimeSpan.FromSeconds(2));
AssertCleanup();
}, _config.First);
RunOn(() =>
{
Sys.ActorOf(Props.Create(() => new ProbeActor(TestActor)), "subject6");
EnterBarrier("actors-started-6");
ExpectMsg("hello6", TimeSpan.FromSeconds(3));
EnterBarrier("watch-established-6");
}, _config.Second);
RunOn(() =>
{
EnterBarrier("actors-started-6");
EnterBarrier("watch-established-6");
}, _config.Third);
EnterBarrier("after-6");
}
public void EventsourcedActors_when_located_at_different_locations_must_track_causality()
{
var probe = CreateTestProbe();
RunOn(() =>
{
var logName = this.LogName();
var endpoint = this.CreateEndpoint(nodeA.Name, ImmutableHashSet <ReplicationConnection> .Empty.Add(Node(nodeB).Address.ToReplicationConnection()));
var actor = Sys.ActorOf(Props.Create(() => new ReplicatedActor("pa", endpoint.Logs[logName], probe.Ref)));
probe.ExpectMsg(("x", VectorTime(0, 1), VectorTime(0, 1)));
actor.Tell("y");
probe.ExpectMsg(("y", VectorTime(2, 1), VectorTime(2, 1)));
EnterBarrier("reply");
TestConductor.Blackhole(nodeA, nodeB, ThrottleTransportAdapter.Direction.Both).Wait();
EnterBarrier("broken");
actor.Tell("z1");
probe.ExpectMsg(("z1", VectorTime(3, 1), VectorTime(3, 1)));
EnterBarrier("repair");
TestConductor.PassThrough(nodeA, nodeB, ThrottleTransportAdapter.Direction.Both).Wait();
probe.ExpectMsg(("z2", VectorTime(2, 3), VectorTime(3, 3)));
}, this.nodeA);
RunOn(() =>
{
var logName = this.LogName();
var endpoint = this.CreateEndpoint(nodeB.Name, ImmutableHashSet <ReplicationConnection> .Empty.Add(Node(nodeA).Address.ToReplicationConnection()));
var actor = Sys.ActorOf(Props.Create(() => new ReplicatedActor("pb", endpoint.Logs[logName], probe.Ref)));
actor.Tell("x");
probe.ExpectMsg(("x", VectorTime(0, 1), VectorTime(0, 1)));
probe.ExpectMsg(("y", VectorTime(2, 1), VectorTime(2, 1)));
EnterBarrier("reply");
EnterBarrier("broken");
actor.Tell("z2");
probe.ExpectMsg(("z2", VectorTime(2, 3), VectorTime(2, 3)));
EnterBarrier("repair");
probe.ExpectMsg(("z1", VectorTime(3, 1), VectorTime(3, 3)));
}, this.nodeB);
EnterBarrier("finish");
}
//[MultiNodeFact(Skip = "TODO")]
public void ClusterClient_should_reestablish_connection_to_receptionist_after_partition()
{
ClusterClient_should_reestablish_connection_to_another_receptionist_when_server_is_shutdown();
Within(TimeSpan.FromSeconds(30), () =>
{
RunOn(() =>
{
var c = Sys.ActorOf(Client.ClusterClient.Props(ClusterClientSettings.Create(Sys).WithInitialContacts(InitialContacts)), "client3");
c.Tell(new Client.ClusterClient.Send("/user/service2", "bonjour2", localAffinity: true));
ExpectMsg("bonjour2-ack");
var lastSenderAddress = LastSender.Path.Address;
var receptionistRoleName = RoleName(lastSenderAddress);
if (receptionistRoleName == null)
{
throw new Exception("Unexpected missing role name: " + lastSenderAddress);
}
// shutdown all but the one that the client is connected to
foreach (var roleName in _remainingServerRoleNames.ToArray())
{
if (!roleName.Equals(receptionistRoleName))
{
TestConductor.Exit(roleName, 0).Wait();
}
}
_remainingServerRoleNames = new HashSet <RoleName>(new[] { receptionistRoleName });
// network partition between client and server
TestConductor.Blackhole(_client, receptionistRoleName, ThrottleTransportAdapter.Direction.Both).Wait();
c.Tell(new Client.ClusterClient.Send("/user/service2", "ping", localAffinity: true));
// if we would use remote watch the failure detector would trigger and
// connection quarantined
ExpectNoMsg(TimeSpan.FromSeconds(5));
TestConductor.PassThrough(_client, receptionistRoleName, ThrottleTransportAdapter.Direction.Both).Wait();
var expectedAddress = Node(receptionistRoleName).Address;
AwaitAssert(() =>
{
c.Tell(new Client.ClusterClient.Send("/user/service2", "bonjour3", localAffinity: true));
ExpectMsg("bonjour3-ack", TimeSpan.FromSeconds(1));
var lastSenderAddress2 = LastSender.Path.Address;
Assert.Equal(expectedAddress, lastSenderAddress2);
});
Sys.Stop(c);
}, _client);
EnterBarrier("after-5");
});
}
public void Must_receive_terminated_when_remote_actor_system_is_restarted()
{
RunOn(() =>
{
var secondAddress = Node(_specConfig.Second).Address;
EnterBarrier("actors-started");
var subject = Identify(_specConfig.Second, "subject");
Watch(subject);
subject.Tell("hello");
ExpectMsg("hello");
EnterBarrier("watch-established");
// simulate a hard shutdown, nothing sent from the shutdown node
TestConductor.Blackhole(_specConfig.Second, _specConfig.First, ThrottleTransportAdapter.Direction.Send)
.GetAwaiter()
.GetResult();
TestConductor.Shutdown(_specConfig.Second).GetAwaiter().GetResult();
ExpectTerminated(subject, TimeSpan.FromSeconds(20));
Within(TimeSpan.FromSeconds(10), () =>
{
// retry because the Subject actor might not be started yet
AwaitAssert(() =>
{
Sys.ActorSelection(new RootActorPath(secondAddress) / "user" /
"subject").Tell("shutdown");
ExpectMsg <string>(msg => { return("shutdown-ack" == msg); }, TimeSpan.FromSeconds(1));
});
});
}, _specConfig.First);
RunOn(() =>
{
var addr = Sys.AsInstanceOf <ExtendedActorSystem>().Provider.DefaultAddress;
Sys.ActorOf(Props.Create(() => new Subject()), "subject");
EnterBarrier("actors-started");
EnterBarrier("watch-established");
Sys.WhenTerminated.Wait(TimeSpan.FromSeconds(30));
var sb = new StringBuilder().AppendLine("akka.remote.helios.tcp {").AppendLine("hostname = " + addr.Host)
.AppendLine("port = " + addr.Port)
.AppendLine("}");
var freshSystem = ActorSystem.Create(Sys.Name,
ConfigurationFactory.ParseString(sb.ToString()).WithFallback(Sys.Settings.Config));
freshSystem.ActorOf(Props.Create(() => new Subject()), "subject");
freshSystem.WhenTerminated.Wait(TimeSpan.FromSeconds(30));
}, _specConfig.Second);
}
private void AttemptSysMsgRedelivery_must_redeliver_system_message_after_inactivity()
{
Sys.ActorOf(Props.Create <AttemptSysMsgRedeliverySpecConfig.Echo>(), "echo");
EnterBarrier("echo-started");
Sys.ActorSelection(Node(_config.First) / "user" / "echo").Tell(new Identify(null));
var firstRef = ExpectMsg <ActorIdentity>().Subject;
Sys.ActorSelection(Node(_config.First) / "user" / "echo").Tell(new Identify(null));
var secondRef = ExpectMsg <ActorIdentity>().Subject;
EnterBarrier("refs-retrieved");
RunOn(() =>
{
TestConductor.Blackhole(_config.First, _config.Second, ThrottleTransportAdapter.Direction.Both).Wait();
}, _config.First);
EnterBarrier("blackhole");
RunOn(() =>
{
Watch(secondRef);
}, _config.First, _config.Third);
RunOn(() =>
{
Watch(firstRef);
}, _config.Second);
EnterBarrier("watch-established");
RunOn(() =>
{
TestConductor.PassThrough(_config.First, _config.Second, ThrottleTransportAdapter.Direction.Both).Wait();
}, _config.First);
EnterBarrier("pass-through");
Sys.ActorSelection("/user/echo").Tell(PoisonPill.Instance);
RunOn(() =>
{
ExpectTerminated(secondRef, 10.Seconds());
}, _config.First, _config.Third);
RunOn(() =>
{
ExpectTerminated(firstRef, 10.Seconds());
}, _config.Second);
EnterBarrier("done");
}
A_heartbeat_driven_Failure_Detector_mark_node_as_unavailable_when_network_partition_and_then_back_to_available_when_partition_is_healed
()
{
RunOn(() => {
TestConductor.Blackhole(_config.First, _config.Second, ThrottleTransportAdapter.Direction.Both).Wait();
}, _config.First);
EnterBarrier("broken");
RunOn(() =>
{
// detect failure...
AwaitCondition(() => !Cluster.FailureDetector.IsAvailable(GetAddress(_config.Second)),
TimeSpan.FromSeconds(15));
// other connections still ok
Cluster.FailureDetector.IsAvailable(GetAddress(_config.Third)).ShouldBeTrue();
}, _config.First);
RunOn(() =>
{
// detect failure...
AwaitCondition(() => !Cluster.FailureDetector.IsAvailable(GetAddress(_config.First)),
TimeSpan.FromSeconds(15));
// other connections still ok
Cluster.FailureDetector.IsAvailable(GetAddress(_config.Third)).ShouldBeTrue();
}, _config.Second);
EnterBarrier("partitioned");
RunOn(() => {
TestConductor.PassThrough(_config.First, _config.Second, ThrottleTransportAdapter.Direction.Both).Wait();
}, _config.First);
EnterBarrier("repaired");
RunOn(() =>
{
AwaitCondition(() => Cluster.FailureDetector.IsAvailable(GetAddress(_config.Second)),
TimeSpan.FromSeconds(15));
}, _config.First, _config.Third);
RunOn(() =>
{
AwaitCondition(() => Cluster.FailureDetector.IsAvailable(GetAddress(_config.First)),
TimeSpan.FromSeconds(15));
}, _config.Second);
EnterBarrier("after-2");
}
public void EventLog_replication_must_detect_replication_server_availability()
{
var probeAvailable1 = CreateTestProbe();
var probeAvailable2 = CreateTestProbe();
var probeUnavailable = CreateTestProbe();
Sys.EventStream.Subscribe(probeAvailable1.Ref, typeof(ReplicationEndpoint.Available));
Sys.EventStream.Subscribe(probeUnavailable.Ref, typeof(ReplicationEndpoint.Unavailable));
EnterBarrier("subscribe");
var logName = this.LogName();
RunOn(() =>
{
this.CreateEndpoint(nodeA.Name, ImmutableHashSet <ReplicationConnection> .Empty.Add(Node(nodeB).Address.ToReplicationConnection()));
probeAvailable1.ExpectMsg(new ReplicationEndpoint.Available(nodeB.Name, logName));
EnterBarrier("connected");
TestConductor.Blackhole(nodeA, nodeB, ThrottleTransportAdapter.Direction.Both).Wait();
var u = probeUnavailable.ExpectMsg <ReplicationEndpoint.Unavailable>();
u.EndpointId.Should().Be(nodeB.Name);
u.Causes.First().Should().BeOfType <ReplicationReadTimeoutException>();
Sys.EventStream.Subscribe(probeAvailable2.Ref, typeof(ReplicationEndpoint.Available));
EnterBarrier("repair");
TestConductor.PassThrough(nodeA, nodeB, ThrottleTransportAdapter.Direction.Both).Wait();
probeAvailable2.ExpectMsg(new ReplicationEndpoint.Available(nodeB.Name, logName));
}, this.nodeA);
RunOn(() =>
{
this.CreateEndpoint(nodeB.Name, ImmutableHashSet <ReplicationConnection> .Empty.Add(Node(nodeA).Address.ToReplicationConnection()));
probeAvailable1.ExpectMsg(new ReplicationEndpoint.Available(nodeA.Name, logName));
EnterBarrier("connected");
var u = probeUnavailable.ExpectMsg <ReplicationEndpoint.Unavailable>();
u.EndpointId.Should().Be(nodeB.Name);
u.Causes.First().Should().BeOfType <ReplicationReadTimeoutException>();
Sys.EventStream.Subscribe(probeAvailable2.Ref, typeof(ReplicationEndpoint.Available));
EnterBarrier("repair");
probeAvailable2.ExpectMsg(new ReplicationEndpoint.Available(nodeB.Name, logName));
}, this.nodeB);
EnterBarrier("finish");
}
public void ClusterSharding_should_failover_shards_on_crashed_node()
{
ClusterSharding_should_support_proxy_only_mode();
Within(TimeSpan.FromSeconds(30), () =>
{
// mute logging of deadLetters during shutdown of systems
if (!Log.IsDebugEnabled)
{
Sys.EventStream.Publish(new Mute(new DeadLettersFilter(new PredicateMatcher(x => true), new PredicateMatcher(x => true))));
}
EnterBarrier("logs-muted");
RunOn(() =>
{
TestConductor.Exit(_second, 0).Wait();
}, _controller);
EnterBarrier("crash-second");
RunOn(() =>
{
var probe1 = CreateTestProbe();
AwaitAssert(() =>
{
Within(TimeSpan.FromSeconds(1), () =>
{
var r = _region.Value;
r.Tell(new Counter.Get(2), probe1.Ref);
probe1.ExpectMsg(4);
Assert.Equal(r.Path / "2" / "2", probe1.LastSender.Path);
});
});
var probe2 = CreateTestProbe();
AwaitAssert(() =>
{
Within(TimeSpan.FromSeconds(1), () =>
{
var r = _region.Value;
r.Tell(new Counter.Get(12), probe2.Ref);
probe2.ExpectMsg(1);
Assert.Equal(r.Path / "0" / "12", probe2.LastSender.Path);
});
});
}, _first);
EnterBarrier("after-6");
});
}
public void ClusterClient_must_reestablish_connection_to_receptionist_after_partition()
{
Within(30.Seconds(), () =>
{
RunOn(() =>
{
var c = Sys.ActorOf(ClusterClient.Props(ClusterClientSettings.Create(Sys).WithInitialContacts(InitialContacts)), "client3");
c.Tell(new ClusterClient.Send("/user/service2", "bonjour2", localAffinity: true));
var reply = ExpectMsg <ClusterClientSpecConfig.Reply>();
reply.Msg.Should().Be("bonjour2-ack");
RoleName receptionistRoleName = GetRoleName(reply.Node);
if (receptionistRoleName == null)
{
throw new Exception("Unexpected missing role name: " + reply.Node);
}
// shutdown all but the one that the client is connected to
_remainingServerRoleNames.Where(r => !r.Equals(receptionistRoleName)).ForEach(r =>
{
TestConductor.Exit(r, 0).Wait();
});
_remainingServerRoleNames = ImmutableHashSet.Create(receptionistRoleName);
// network partition between client and server
TestConductor.Blackhole(_config.Client, receptionistRoleName, ThrottleTransportAdapter.Direction.Both).Wait();
c.Tell(new ClusterClient.Send("/user/service2", "ping", localAffinity: true));
// if we would use remote watch the failure detector would trigger and
// connection quarantined
ExpectNoMsg(5.Seconds());
TestConductor.PassThrough(_config.Client, receptionistRoleName, ThrottleTransportAdapter.Direction.Both).Wait();
var expectedAddress = GetAddress(receptionistRoleName);
AwaitAssert(() =>
{
var probe = CreateTestProbe();
c.Tell(new ClusterClient.Send("/user/service2", "bonjour3", localAffinity: true), probe.Ref);
var reply2 = probe.ExpectMsg <ClusterClientSpecConfig.Reply>(1.Seconds());
reply2.Msg.Should().Be("bonjour3-ack");
reply2.Node.Should().Be(expectedAddress);
});
Sys.Stop(c);
}, _config.Client);
EnterBarrier("after-5");
});
}
public void LeaseMajority_in_a_5_node_cluster_should_keep_the_side_that_can_acquire_the_lease_round_2()
{
var lease = TestLeaseExt.Get(Sys).GetTestLease(testLeaseName);
RunOn(() =>
{
lease.SetNextAcquireResult(Task.FromResult(true));
}, _config.Node1);
RunOn(() =>
{
lease.SetNextAcquireResult(Task.FromResult(false));
}, _config.Node2, _config.Node3);
EnterBarrier("lease-in-place-2");
RunOn(() =>
{
foreach (var x in new[] { _config.Node1 })
{
foreach (var y in new[] { _config.Node2, _config.Node3 })
{
TestConductor.Blackhole(x, y, ThrottleTransportAdapter.Direction.Both).Wait();
}
}
}, _config.Node1);
EnterBarrier("blackholed-clean-partition-2");
RunOn(() =>
{
Within(TimeSpan.FromSeconds(20), () =>
{
AwaitAssert(() =>
{
Cluster.State.Members.Count.Should().Be(1);
});
});
}, _config.Node1);
RunOn(() =>
{
Within(TimeSpan.FromSeconds(20), () =>
{
AwaitAssert(() =>
{
Cluster.IsTerminated.Should().BeTrue();
});
});
}, _config.Node2, _config.Node3);
EnterBarrier("done-2");
}
public void ANewRemoteActorMustBeLocallyInstantiatedOnARemoteNodeWithNullParameterAndBeAbleToCommunicateThroughItsRemoteActorRef()
{
RunOn(() =>
{
var actor = Sys.ActorOf(Props.Create(() => new NewRemoteActorMultiNodeSpecConfig.SomeActorWithParam(null)),
"service-hello2");
var foo = Assert.IsType <RemoteActorRef>(actor);
actor.Path.Address.ShouldBe(Node(_config.Slave).Address);
var slaveAddress = TestConductor.GetAddressFor(_config.Slave).Result;
actor.Tell("identify");
ExpectMsg <IActorRef>().Path.Address.ShouldBe(slaveAddress);
}, _config.Master);
EnterBarrier("done");
}
public void Cluster_of_2_nodes_must_become_singleton_cluster_when_one_node_is_shutdown()
{
RunOn(() =>
{
var secondAddress = GetAddress(_config.Second);
TestConductor.Exit(_config.Second, 0).Wait();
MarkNodeAsUnavailable(secondAddress);
AwaitMembersUp(1, ImmutableHashSet.Create <Address>(secondAddress), TimeSpan.FromSeconds(30));
ClusterView.IsSingletonCluster.ShouldBeTrue();
AwaitCondition(() => ClusterView.IsLeader);
}, _config.First);
EnterBarrier("after-3");
}
public void A_new_remote_actor_must_be_locally_instantiated_on_a_remote_node_with_null_parameter_and_be_able_to_communicate_through_its_remote_actor_ref()
{
RunOn(() =>
{
var actor = Sys.ActorOf(Props.Create(() => new NewRemoteActorMultiNodeSpecConfig.SomeActorWithParam(null)),
"service-hello2");
var foo = Assert.IsType <RemoteActorRef>(actor);
actor.Path.Address.ShouldBe(Node(_config.Slave).Address);
var slaveAddress = TestConductor.GetAddressFor(_config.Slave).Result;
actor.Tell("identify");
ExpectMsg <IActorRef>().Path.Address.ShouldBe(slaveAddress);
}, _config.Master);
EnterBarrier("done");
}
public void A_Network_partition_tolerant_cluster_must_heal_two_isolated_islands()
{
Within(45.Seconds(), () =>
{
var island1 = ImmutableArray.Create(_config.First, _config.Second);
var island2 = ImmutableArray.Create(_config.Third, _config.Fourth, _config.Fifth);
RunOn(() =>
{
// split the cluster in two parts (first, second) / (third, fourth, fifth)
foreach (var role1 in island1)
{
foreach (var role2 in island2)
{
TestConductor.Blackhole(role1, role2, ThrottleTransportAdapter.Direction.Both).Wait();
}
}
}, _config.First);
EnterBarrier("blackhole-4");
RunOn(() =>
{
AssertUnreachable(island2.ToArray());
}, island1.ToArray());
RunOn(() =>
{
AssertUnreachable(island1.ToArray());
}, island2.ToArray());
EnterBarrier("unreachable-4");
RunOn(() =>
{
foreach (var role1 in island1)
{
foreach (var role2 in island2)
{
TestConductor.PassThrough(role1, role2, ThrottleTransportAdapter.Direction.Both).Wait();
}
}
}, _config.First);
EnterBarrier("repair-4");
AssertCanTalk(island1.AddRange(island2).ToArray());
});
}
public void Cluster_sharding_with_remember_entities_should_start_remembered_entities_when_coordinator_fail_over()
{
Within(TimeSpan.FromSeconds(30), () =>
{
Join(_config.Second, _config.Second);
RunOn(() =>
{
StartSharding(Sys, TestActor);
_region.Value.Tell(1);
ExpectMsg <Started>();
}, _config.Second);
EnterBarrier("second-started");
Join(_config.Third, _config.Second);
RunOn(() =>
{
StartSharding(Sys, TestActor);
}, _config.Third);
RunOn(() =>
{
Within(Remaining, () =>
{
AwaitAssert(() =>
{
Cluster.State.Members.Count.Should().Be(2);
Cluster.State.Members.Should().OnlyContain(i => i.Status == MemberStatus.Up);
});
});
}, _config.Second, _config.Third);
EnterBarrier("all-up");
RunOn(() =>
{
TestConductor.Exit(_config.Second, 0).Wait();
}, _config.First);
EnterBarrier("crash-second");
RunOn(() =>
{
ExpectMsg <Started>(Remaining);
}, _config.Third);
EnterBarrier("after-2");
});
}
public void A_Cluster_of_6_nodes_with_monitored_by_nr_of_members_2_must_remove_all_shutdown_nodes()
{
var others = Roles.Drop(1).ToList();
var shutdownAddresses = others.Select(c => GetAddress(c)).ToImmutableHashSet();
EnterBarrier("before-all-other-shutdown");
RunOn(() =>
{
foreach (var node in others)
{
TestConductor.Exit(node, 0).Wait();
}
}, _config.First);
EnterBarrier("all-other-shutdown");
AwaitMembersUp(numbersOfMembers: 1, canNotBePartOfMemberRing: shutdownAddresses, timeout: 30.Seconds());
}
public void ClusterClient_must_reestablish_connection_to_receptionist_after_server_restart()
{
Within(30.Seconds(), () =>
{
RunOn(() =>
{
_remainingServerRoleNames.Count.Should().Be(1);
var remainingContacts = _remainingServerRoleNames.Select(r => Node(r) / "system" / "receptionist").ToImmutableHashSet();
var c = Sys.ActorOf(ClusterClient.Props(ClusterClientSettings.Create(Sys).WithInitialContacts(remainingContacts)), "client4");
c.Tell(new ClusterClient.Send("/user/service2", "bonjour4", localAffinity: true));
var reply = ExpectMsg <ClusterClientSpecConfig.Reply>(10.Seconds());
reply.Msg.Should().Be("bonjour4-ack");
reply.Node.Should().Be(remainingContacts.First().Address);
// TODO: bug, cannot compare with a logsource
var logSource = $"{Sys.AsInstanceOf<ExtendedActorSystem>().Provider.DefaultAddress}/user/client4";
EventFilter.Info(start: "Connected to").ExpectOne(() =>
{
EventFilter.Info(start: "Lost contact").ExpectOne(() =>
{
// shutdown server
TestConductor.Shutdown(_remainingServerRoleNames.First()).Wait();
});
});
c.Tell(new ClusterClient.Send("/user/service2", "shutdown", localAffinity: true));
Thread.Sleep(2000); // to ensure that it is sent out before shutting down system
}, _config.Client);
RunOn(() =>
{
Sys.WhenTerminated.Wait(20.Seconds());
// start new system on same port
var port = Cluster.Get(Sys).SelfAddress.Port;
var sys2 = ActorSystem.Create(
Sys.Name,
ConfigurationFactory.ParseString($"akka.remote.dot-netty.tcp.port={port}").WithFallback(Sys.Settings.Config));
Cluster.Get(sys2).Join(Cluster.Get(sys2).SelfAddress);
var service2 = sys2.ActorOf(Props.Create(() => new ClusterClientSpecConfig.TestService(TestActor)), "service2");
ClusterClientReceptionist.Get(sys2).RegisterService(service2);
sys2.WhenTerminated.Wait(20.Seconds());
}, _remainingServerRoleNames.ToArray());
});
}
public void RedeliverSystemMessageAfterInactivity()
{
var echo = ActorOf <AttemptSysMsgRedeliveryMultiNetSpec.Echo>("echo");
EnterBarrier("echo-started");
Sys.ActorSelection(Node(_config.First) / "user" / "echo").Tell(new Identify(null));
var firstRef = ExpectMsg <ActorIdentity>().Subject;
Sys.ActorSelection(Node(_config.Second) / "user" / "echo").Tell(new Identify(null));
var secondRef = ExpectMsg <ActorIdentity>().Subject;
EnterBarrier("refs-retrieved");
RunOn(() =>
TestConductor.Blackhole(_config.First, _config.Second, ThrottleTransportAdapter.Direction.Both)
.Wait(),
_config.First);
EnterBarrier("blackhole");
RunOn(() => Watch(secondRef),
_config.First, _config.Third);
RunOn(() => Watch(firstRef),
_config.Second);
EnterBarrier("watch-established");
RunOn(() =>
TestConductor.PassThrough(_config.First, _config.Second, ThrottleTransportAdapter.Direction.Both)
.Wait(),
_config.First);
EnterBarrier("pass-through");
Sys.ActorSelection("/user/echo").Tell(PoisonPill.Instance);
RunOn(() => ExpectTerminated(secondRef, TimeSpan.FromSeconds(10)),
_config.First, _config.Third);
RunOn(() => ExpectTerminated(firstRef, TimeSpan.FromSeconds(10)),
_config.Second);
EnterBarrier("done");
}
public void ValidatingConductorAndScreen()
{
var conductor = new TestConductor();
Assert.IsFalse(string.IsNullOrWhiteSpace(conductor.Error));
Assert.IsTrue(conductor.HasError);
Assert.IsNotNull(conductor["TestInt"]);
Assert.IsTrue(conductor["TestInt"].Contains("TestInt"));
conductor.ActiveItem.TestInt = 100;
Assert.IsTrue(string.IsNullOrWhiteSpace(conductor["TestInt"]));
Assert.IsTrue(conductor["TestString"].Contains("TestString"));
conductor.ActiveItem.TestString = "100";
Assert.IsTrue(string.IsNullOrWhiteSpace(conductor["TestInt"]));
Assert.IsTrue(string.IsNullOrWhiteSpace(conductor["TestString"]));
Assert.IsTrue(string.IsNullOrWhiteSpace(conductor.Error));
Assert.IsFalse(conductor.HasError);
}
public void Cluster_of_5_members_must_detect_network_partition_and_mark_nodes_on_other_side_as_unreachable_and_form_new_cluster()
{
EnterBarrier("before-split");
RunOn(() =>
{
// split the cluster in two parts (first, second) / (third, fourth, fifth)
foreach (var role1 in side1)
{
foreach (var role2 in side2)
{
TestConductor.Blackhole(role1, role2, ThrottleTransportAdapter.Direction.Both).Wait();
}
}
}, _config.First);
EnterBarrier("after-split");
RunOn(() =>
{
foreach (var role in side2)
{
MarkNodeAsUnavailable(GetAddress(role));
}
// auto-down
AwaitMembersUp(side1.Count, side2.Select(r => GetAddress(r)).ToImmutableHashSet());
AssertLeader(side1.ToArray());
}, side1.ToArray());
RunOn(() =>
{
foreach (var role in side1)
{
MarkNodeAsUnavailable(GetAddress(role));
}
// auto-down
AwaitMembersUp(side2.Count, side1.Select(r => GetAddress(r)).ToImmutableHashSet());
AssertLeader(side2.ToArray());
}, side2.ToArray());
EnterBarrier("after-2");
}
public void An_actor_system_that_deploys_actors_on_another_node_must_be_able_to_shutdown_when_remote_node_crash()
{
RunOn(() =>
{
var hello = Sys.ActorOf(Props.Create(() => new Hello()), "hello");
hello.Path.Address.ShouldBe(Node(_specConfig.Third).Address);
EnterBarrier("hello-deployed");
EnterBarrier("third-crashed");
Sleep();
// if the remote deployed actor is not removed the system will not shutdown
var timeOut = RemainingOrDefault;
try
{
Sys.WhenTerminated.Wait(timeOut);
}
catch (TimeoutException ex)
{
//TODO: add printTree
_assertions.Fail("Failed to stop {0} within {1} ", Sys.Name, timeOut);
}
}, _specConfig.Second);
RunOn(() =>
{
EnterBarrier("hello-deployed");
EnterBarrier("third-crashed");
}, _specConfig.Third);
RunOn(() =>
{
EnterBarrier("hello-deployed");
Sleep();
TestConductor.Exit(_specConfig.Third, 0).GetAwaiter().GetResult();
EnterBarrier("third-crashed");
//second system will be shutdown
TestConductor.Shutdown(_specConfig.Second).GetAwaiter().GetResult();
EnterBarrier("after-3");
}, _specConfig.First);
}
private void A_Cluster_must_detect_network_partition_and_down_minor_part_of_the_cluster()
{
var majority = new[] { _config.First, _config.Second, _config.Third };
var minority = new[] { _config.Fourth, _config.Fifth };
EnterBarrier("before-split");
var downed = false;
RunOn(() =>
{
Cluster.RegisterOnMemberRemoved(() => downed = true);
}, minority);
RunOn(() =>
{
foreach (var a in majority)
{
foreach (var b in minority)
{
TestConductor.Blackhole(a, b, ThrottleTransportAdapter.Direction.Both).Wait();
}
}
}, _config.First);
EnterBarrier("after-split");
RunOn(() =>
{
// side with majority of the nodes must stay up
AwaitMembersUp(majority.Length, canNotBePartOfMemberRing: minority.Select(GetAddress).ToImmutableHashSet());
AssertLeader(majority);
}, majority);
RunOn(() =>
{
// side with majority of the nodes must stay up, minority must go down
AwaitAssert(() => downed.Should().BeTrue("cluster node on-removed hook has been triggered"));
}, minority);
EnterBarrier("after-2");
}
A_heartbeat_driven_Failure_Detector_mark_node_as_unavailable_if_a_node_in_the_cluster_is_shut_down_and_its_heartbeats_stops
()
{
RunOn(() => {
TestConductor.Exit(_config.Third, 0).Wait();
}, _config.First);
EnterBarrier("third-shutdown");
RunOn(() =>
{
// remaining nodes should detect failure...
AwaitCondition(() => !Cluster.FailureDetector.IsAvailable(GetAddress(_config.Third)), TimeSpan.FromSeconds(15));
// other connections still ok
Cluster.FailureDetector.IsAvailable(GetAddress(_config.First)).ShouldBeTrue();
Cluster.FailureDetector.IsAvailable(GetAddress(_config.Second)).ShouldBeTrue();
}, _config.First, _config.Second);
EnterBarrier("after-3");
}
public void Remoting_must_lookup_remote_actor()
{
RunOn(
() =>
{
Sys.ActorSelection(Node(_config.Master) / "user" / "service-hello")
.Tell(new Identify("id1"));
var hello = ExpectMsg <ActorIdentity>()
.Subject;
Assert.IsType <RemoteActorRef>(hello);
var masterAddress = TestConductor.GetAddressFor(_config.Master).Result;
Assert.StrictEqual(masterAddress, hello.Path.Address);
},
_config.Slave);
EnterBarrier("done");
}
protected void AttachConductor(TestConductor tc)
{
var timeout = tc.Settings.BarrierTimeout;
try
{
//TODO: Async stuff
if (SelfIndex == 0)
tc.StartController(InitialParticipants, _myself, _controllerAddr).Wait(timeout);
else
tc.StartClient(_myself, _controllerAddr).Wait(timeout);
}
catch (Exception e)
{
throw new Exception("failure while attaching new conductor", e);
}
TestConductor = tc;
}
