public void HandleFailure(ISupervisor supervisor, PID child, RestartStatistics rs, Exception cause, object message)
{
if (ShouldStop(rs))
{
_logger.LogWarning($"Stopping {child.ToShortString()} connection to address {_address} after retries expired Reason {cause}");
_cancelFutureRetries.Cancel();
supervisor.StopChildren(child);
ProcessRegistry.Instance.Remove(child); //TODO: work out why this hangs around in the process registry
var terminated = new EndpointTerminatedEvent
{
Address = _address
};
Actor.EventStream.Publish(terminated);
//Reset everything to original values
_backoff = config.EndpointWriterOptions.RetryBackOffms;
_cancelFutureRetries = new CancellationTokenSource();
}
else
{
_backoff = _backoff * 2;
var noise = _random.Next(_backoff);
var duration = TimeSpan.FromMilliseconds(_backoff + noise);
Task.Delay(duration).ContinueWith(t =>
{
_logger.LogWarning($"Restarting {child.ToShortString()} for {_address} after {duration} Reason {cause}");
supervisor.RestartChildren(cause, child);
}, _cancelFutureRetries.Token);
}
}
private void Notify(MemberStatus @new, MemberStatus old)
{
if (@new == null && old == null)
{
return; //ignore
}
if (@new == null)
{
//notify left
var left = new MemberLeftEvent(old.Host, old.Port, old.Kinds);
Actor.EventStream.Publish(left);
_members.Remove(old.Address);
var endpointTerminated = new EndpointTerminatedEvent
{
Address = old.Address
};
Actor.EventStream.Publish(endpointTerminated);
return;
}
if (old == null)
{
//notify joined
var joined = new MemberJoinedEvent(@new.Host, @new.Port, @new.Kinds);
Actor.EventStream.Publish(joined);
return;
}
if (@new.MemberId != old.MemberId)
{
var rejoined = new MemberRejoinedEvent(@new.Host, @new.Port, @new.Kinds);
Actor.EventStream.Publish(rejoined);
return;
}
if (old.Alive && [email protected])
{
var unavailable = new MemberUnavailableEvent(@new.Host, @new.Port, @new.Kinds);
Actor.EventStream.Publish(unavailable);
return;
}
if (@new.Alive && !old.Alive)
{
var available = new MemberAvailableEvent(@new.Host, @new.Port, @new.Kinds);
Actor.EventStream.Publish(available);
}
}
private static void UpdateAndNotify(MemberStatus @new, MemberStatus old)
{
if (@new == null && old == null)
{
return; //ignore
}
if (@new == null)
{
//update MemberStrategy
foreach (var k in old.Kinds)
{
if (MemberStrategyByKind.TryGetValue(k, out var ms))
{
ms.RemoveMember(old);
if (ms.GetAllMembers().Count == 0)
{
MemberStrategyByKind.Remove(k);
}
}
}
//notify left
var left = new MemberLeftEvent(old.Host, old.Port, old.Kinds);
Actor.EventStream.Publish(left);
Members.Remove(old.Address);
var endpointTerminated = new EndpointTerminatedEvent
{
Address = old.Address
};
Actor.EventStream.Publish(endpointTerminated);
return;
}
if (old == null)
{
//update MemberStrategy
foreach (var k in @new.Kinds)
{
if (!MemberStrategyByKind.ContainsKey(k))
{
MemberStrategyByKind[k] = Cluster.Config.MemberStrategyBuilder(k);
}
MemberStrategyByKind[k].AddMember(@new);
}
//notify joined
var joined = new MemberJoinedEvent(@new.Host, @new.Port, @new.Kinds);
Actor.EventStream.Publish(joined);
return;
}
//update MemberStrategy
if (@new.Alive != old.Alive || @new.MemberId != old.MemberId || @new.StatusValue != null && [email protected](old.StatusValue))
{
foreach (var k in @new.Kinds)
{
if (MemberStrategyByKind.TryGetValue(k, out var ms))
{
ms.UpdateMember(@new);
}
}
}
//notify changes
if (@new.MemberId != old.MemberId)
{
var rejoined = new MemberRejoinedEvent(@new.Host, @new.Port, @new.Kinds);
Actor.EventStream.Publish(rejoined);
}
}
