private bool RequestRestartPermission(RestartStatistics rs)
{
if (_maxNrOfRetries == 0)
{
return(false);
}
rs.Fail();
if (_withinTimeSpan == null || rs.IsWithinDuration(_withinTimeSpan.Value))
{
return(rs.FailureCount <= _maxNrOfRetries);
}
rs.Reset();
return(true);
}
public void HandleFailure(
ISupervisor supervisor,
PID child,
RestartStatistics rs,
Exception reason,
object?message
)
{
var directive = _decider(child, reason);
switch (directive)
{
case SupervisorDirective.Resume:
LogInfo("Resuming");
supervisor.ResumeChildren(child);
break;
case SupervisorDirective.Restart:
if (ShouldStop(rs))
{
LogInfo("Stopping");
supervisor.StopChildren(supervisor.Children.ToArray());
}
else
{
LogInfo("Restarting");
supervisor.RestartChildren(reason, supervisor.Children.ToArray());
}
break;
case SupervisorDirective.Stop:
LogInfo("Stopping");
supervisor.StopChildren(supervisor.Children.ToArray());
break;
case SupervisorDirective.Escalate:
supervisor.EscalateFailure(reason, message);
break;
default:
throw new ArgumentOutOfRangeException();
}
void LogInfo(string action) => Logger.LogInformation("{Action} {Actor} because of {Reason}", action,
child, reason
);
}
private bool ShouldStop(RestartStatistics rs)
{
if (_maxNrOfRetries == 0)
{
return(true);
}
rs.Fail();
if (rs.NumberOfFailures(_withinTimeSpan) > _maxNrOfRetries)
{
rs.Reset();
return(true);
}
return(false);
}
public void EscalateFailure(Exception reason, object message)
{
if (_restartStatistics == null)
{
_restartStatistics = new RestartStatistics(0, null);
}
var failure = new Failure(Self, reason, _restartStatistics);
if (Parent == null)
{
HandleRootFailure(failure);
}
else
{
Self.SendSystemMessage(SuspendMailbox.Instance);
Parent.SendSystemMessage(failure);
}
}
public void HandleFailure(ISupervisor supervisor, PID child, RestartStatistics rs, Exception reason, object message)
{
if (rs.NumberOfFailures(_backoffWindow) == 0)
{
rs.Reset();
}
rs.Fail();
var backoff = rs.FailureCount * ToNanoseconds(_initialBackoff);
var noise = _random.Next(500);
var duration = TimeSpan.FromMilliseconds(ToMilliseconds(backoff + noise));
Task.Delay(duration).ContinueWith(t =>
{
supervisor.RestartChildren(reason, child);
});
}
//public bool RequestRestartPermission(int maxNrOfRetries, TimeSpan? withinTimeSpan)
//{
// if (maxNrOfRetries == 0)
// {
// return false;
// }
// FailureCount++;
// //supervisor says child may restart, and we don't care about any timewindow
// if (withinTimeSpan == null)
// {
// return FailureCount <= maxNrOfRetries;
// }
// var max = DateTime.Now - withinTimeSpan;
// if (LastFailureTime > max)
// {
// return FailureCount <= maxNrOfRetries;
// }
// //we are past the time limit, we can safely reset the failure count and restart
// FailureCount = 0;
// return true;
//}
public void HandleFailure(ISupervisor supervisor, PID child, RestartStatistics crs, Exception reason)
{
var directive = _decider(child, reason);
switch (directive)
{
case SupervisorDirective.Resume:
//resume the failing child
child.SendSystemMessage(ResumeMailbox.Instance);
break;
case SupervisorDirective.Restart:
//restart the failing child
if (crs.RequestRestartPermission(_maxNrOfRetries, _withinTimeSpan))
{
Console.WriteLine($"Restarting {child.ToShortString()} Reason {reason}");
child.SendSystemMessage(Restart.Instance);
}
else
{
Console.WriteLine($"Stopping {child.ToShortString()} Reason { reason}");
child.Stop();
}
break;
case SupervisorDirective.Stop:
//stop the failing child
Console.WriteLine($"Stopping {child.ToShortString()} Reason {reason}");
child.Stop();
break;
case SupervisorDirective.Escalate:
supervisor.EscalateFailure(child, reason);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public void HandleFailure(ISupervisor supervisor, PID child, RestartStatistics rs, Exception reason)
{
var directive = _decider(child, reason);
switch (directive)
{
case SupervisorDirective.Resume:
Logger.LogInformation($"Resuming {child.ToShortString()} Reason {reason}");
supervisor.ResumeChildren(child);
break;
case SupervisorDirective.Restart:
if (RequestRestartPermission(rs))
{
Logger.LogInformation($"Restarting {child.ToShortString()} Reason {reason}");
supervisor.RestartChildren(reason, supervisor.Children.ToArray());
}
else
{
Logger.LogInformation($"Stopping {child.ToShortString()} Reason { reason}");
supervisor.StopChildren(supervisor.Children.ToArray());
}
break;
case SupervisorDirective.Stop:
Logger.LogInformation($"Stopping {child.ToShortString()} Reason {reason}");
supervisor.StopChildren(supervisor.Children.ToArray());
break;
case SupervisorDirective.Escalate:
supervisor.EscalateFailure(reason, child);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
public Failure(PID who, Exception reason, RestartStatistics crs)
{
Who = who;
Reason = reason;
RestartStatistics = crs;
}
public void HandleFailure(ISupervisor supervisor, PID child, RestartStatistics rs, Exception reason, object message)
{
//always restart
supervisor.RestartChildren(reason, child);
}