/// <summary>
/// It's a delicate difference between <see cref="CoordinatedShutdown.PhaseClusterExiting"/> and <see cref="ClusterEvent.MemberExited"/>.
///
/// MemberExited event is published immediately (leader may have performed that transition on other node),
/// and that will trigger run of <see cref="CoordinatedShutdown"/>, while PhaseClusterExiting will happen later.
/// Using PhaseClusterExiting in the singleton because the graceful shutdown of sharding region
/// should preferably complete before stopping the singleton sharding coordinator on same node.
/// </summary>
private void SetupCoordinatedShutdown()
{
var self = Self;
_coordShutdown.AddTask(CoordinatedShutdown.PhaseClusterExiting, "singleton-exiting-1", () =>
{
var timeout = _coordShutdown.Timeout(CoordinatedShutdown.PhaseClusterExiting);
return(self.Ask(SelfExiting.Instance, timeout).ContinueWith(tr => Done.Instance));
});
}
private void SetupCoordinatedShutdown()
{
var self = Self;
_coordShutdown.AddTask(CoordinatedShutdown.PhaseClusterShardingShutdownRegion, "region-shutdown", () =>
{
self.Tell(GracefulShutdown.Instance);
return(_gracefulShutdownProgress.Task);
});
}
public void CoordinatedShutdown_must_continue_after_timeout_or_failure()
{
var phases = new Dictionary <string, Phase>()
{
{ "a", EmptyPhase },
{ "b", new Phase(ImmutableHashSet <string> .Empty.Add("a"), TimeSpan.FromMilliseconds(100), true) },
{ "c", Phase("b", "a") }
};
var co = new CoordinatedShutdown(ExtSys, phases);
co.AddTask("a", "a1", () =>
{
TestActor.Tell("A");
return(TaskEx.FromException <Done>(new Exception("boom")));
});
co.AddTask("a", "a2", () =>
{
Task.Delay(TimeSpan.FromMilliseconds(100)).Wait();
TestActor.Tell("A");
return(TaskEx.Completed);
});
co.AddTask("b", "b1", () =>
{
TestActor.Tell("B");
return(new TaskCompletionSource <Done>().Task); // never completed
});
co.AddTask("c", "c1", () =>
{
TestActor.Tell("C");
return(TaskEx.Completed);
});
co.Run(CoordinatedShutdown.UnknownReason.Instance).Wait(RemainingOrDefault);
ExpectMsg("A");
ExpectMsg("A");
ExpectMsg("B");
ExpectMsg("C");
}
public void CoordinatedShutdown_must_run_ordered_phases()
{
var phases = new Dictionary <string, Phase>()
{
{ "a", EmptyPhase },
{ "b", Phase("a") },
{ "c", Phase("b", "a") }
};
var co = new CoordinatedShutdown(ExtSys, phases);
co.AddTask("a", "a1", () =>
{
TestActor.Tell("A");
return(TaskEx.Completed);
});
co.AddTask("b", "b1", () =>
{
TestActor.Tell("B");
return(TaskEx.Completed);
});
co.AddTask("b", "b2", () =>
{
// to verify that c is not performed before b
Task.Delay(TimeSpan.FromMilliseconds(100)).Wait();
TestActor.Tell("B");
return(TaskEx.Completed);
});
co.AddTask("c", "c1", () =>
{
TestActor.Tell("C");
return(TaskEx.Completed);
});
co.Run(CoordinatedShutdown.UnknownReason.Instance).Wait(RemainingOrDefault);
ReceiveN(4).Should().Equal(new object[] { "A", "B", "B", "C" });
}
public void CoordinatedShutdown_must_abort_if_recover_is_off()
{
var phases = new Dictionary <string, Phase>()
{
{ "b", new Phase(ImmutableHashSet <string> .Empty.Add("a"), TimeSpan.FromMilliseconds(100), false) },
{ "c", Phase("b", "a") }
};
var co = new CoordinatedShutdown(ExtSys, phases);
co.AddTask("b", "b1", () =>
{
TestActor.Tell("B");
return(new TaskCompletionSource <Done>().Task); // never completed
});
co.AddTask("c", "c1", () =>
{
TestActor.Tell("C");
return(TaskEx.Completed);
});
var result = co.Run(CoordinatedShutdown.UnknownReason.Instance);
ExpectMsg("B");
Intercept <AggregateException>(() =>
{
if (result.Wait(RemainingOrDefault))
{
result.Exception?.Flatten().InnerException.Should().BeOfType <TimeoutException>();
}
else
{
throw new Exception("CoordinatedShutdown task did not complete");
}
});
ExpectNoMsg(TimeSpan.FromMilliseconds(200)); // C not run
}
/// <summary>
/// It's a delicate difference between <see cref="CoordinatedShutdown.PhaseClusterExiting"/> and <see cref="ClusterEvent.MemberExited"/>.
///
/// MemberExited event is published immediately (leader may have performed that transition on other node),
/// and that will trigger run of <see cref="CoordinatedShutdown"/>, while PhaseClusterExiting will happen later.
/// Using PhaseClusterExiting in the singleton because the graceful shutdown of sharding region
/// should preferably complete before stopping the singleton sharding coordinator on same node.
/// </summary>
private void SetupCoordinatedShutdown()
{
var self = Self;
_coordShutdown.AddTask(CoordinatedShutdown.PhaseClusterExiting, "singleton-exiting-1", () =>
{
if (_cluster.IsTerminated || _cluster.SelfMember.Status == MemberStatus.Down)
{
return(Task.FromResult(Done.Instance));
}
else
{
var timeout = _coordShutdown.Timeout(CoordinatedShutdown.PhaseClusterExiting);
return(self.Ask(SelfExiting.Instance, timeout).ContinueWith(tr => Done.Instance));
}
});
}
private void SetupCoordinatedShutdown()
{
var self = Self;
_coordShutdown.AddTask(CoordinatedShutdown.PhaseClusterShardingShutdownRegion, "region-shutdown", () =>
{
if (Cluster.IsTerminated || Cluster.SelfMember.Status == MemberStatus.Down)
{
return(Task.FromResult(Done.Instance));
}
else
{
self.Tell(GracefulShutdown.Instance);
return(_gracefulShutdownProgress.Task);
}
});
}
public void CoordinatedShutdown_must_only_run_once()
{
var phases = new Dictionary <string, Phase>()
{
{ "a", EmptyPhase }
};
var co = new CoordinatedShutdown(ExtSys, phases);
co.AddTask("a", "a1", () =>
{
TestActor.Tell("A");
return(TaskEx.Completed);
});
co.Run().Wait(RemainingOrDefault);
ExpectMsg("A");
co.Run().Wait(RemainingOrDefault);
TestActor.Tell("done");
ExpectMsg("done"); // no additional A
}
public void CoordinatedShutdown_must_only_run_once()
{
var phases = new Dictionary <string, Phase>()
{
{ "a", EmptyPhase }
};
var co = new CoordinatedShutdown(ExtSys, phases);
co.AddTask("a", "a1", () =>
{
TestActor.Tell("A");
return(TaskEx.Completed);
});
co.ShutdownReason.Should().BeNull();
co.Run(customReason).Wait(RemainingOrDefault);
co.ShutdownReason.ShouldBeEquivalentTo(customReason);
ExpectMsg("A");
co.Run(CoordinatedShutdown.UnknownReason.Instance).Wait(RemainingOrDefault);
TestActor.Tell("done");
ExpectMsg("done"); // no additional A
co.ShutdownReason.ShouldBeEquivalentTo(customReason);
}