private async Task HeartBeatLoop()
{
await Task.Yield();
while (!_cluster.System.Shutdown.IsCancellationRequested)
{
try
{
await Task.Delay(_cluster.Config.HeartBeatInterval);
var members = _cluster.MemberList.GetAllMembers();
foreach (var member in members)
{
var pid = PID.FromAddress(member.Address, ClusterHeartBeatName);
try
{
await _context.RequestAsync <HeartbeatResponse>(pid, new HeartbeatRequest(),
TimeSpan.FromSeconds(5)
);
_logger.LogDebug("Heartbeat request for member id {MemberId} Address {Address} succeeded",
member.Id, member.Address
);
}
catch (TimeoutException)
{
if (_cluster.System.Shutdown.IsCancellationRequested)
{
return;
}
_logger.LogWarning("Heartbeat request for member id {MemberId} Address {Address} timed out",
member.Id, member.Address
);
}
catch (DeadLetterException)
{
if (_cluster.System.Shutdown.IsCancellationRequested)
{
return;
}
_logger.LogWarning(
"Heartbeat request for member id {MemberId} Address {Address} got dead letter response",
member.Id, member.Address
);
}
}
}
catch (Exception x)
{
_logger.LogError(x, "Heartbeat loop failed");
}
}
}
public async Task <StoredActivation?> TryGetExistingActivation(
ClusterIdentity clusterIdentity,
CancellationToken ct
)
{
var pidLookup = await LookupKey(GetKey(clusterIdentity), ct);
return(pidLookup?.Address == null || pidLookup?.UniqueIdentity == null
? null
: new StoredActivation(pidLookup.MemberId !,
PID.FromAddress(pidLookup.Address, pidLookup.UniqueIdentity)
));
}
private void Invalidate(ClusterIdentity identity, PID activation, BitArray activeRemotes)
{
var message = new ActivationTerminated
{
ClusterIdentity = identity,
Pid = activation
};
var remotesToInvalidate = Cluster.MemberList.GetAllMembers()
.Select(m => Cluster.MemberList.GetMetaMember(m.Id))
.Where(m => activeRemotes.Length > m.Index && activeRemotes[m.Index])
.Select(m => m.Member.Address);
foreach (var address in remotesToInvalidate)
{
Cluster.System.Root.Send(PID.FromAddress(address, ActorName), message);
}
}
static PID ActivatorForAddress(string address) => PID.FromAddress(address, "activator");
public override async Task Receive(
IAsyncStreamReader <MessageBatch> requestStream,
IServerStreamWriter <Unit> responseStream,
ServerCallContext context
)
{
using var cancellationTokenRegistration = _endpointManager.CancellationToken.Register(() => {
Logger.LogDebug("[EndpointReader] Telling to {Address} to stop", context.Peer);
try
{
_ = responseStream.WriteAsync(new Unit());
}
catch (Exception e)
{
Logger.LogError(e, "[EndpointReader] Didn't tell to {Address} to stop", context.Peer);
}
}
);
var targets = new PID[100];
while (await requestStream.MoveNext(context.CancellationToken).ConfigureAwait(false))
{
if (_endpointManager.CancellationToken.IsCancellationRequested)
{
// We read all the messages ignoring them to gracefully end the request
continue;
}
var batch = requestStream.Current;
// Logger.LogDebug("[EndpointReader] Received a batch of {Count} messages from {Remote}",
// batch.TargetNames.Count, context.Peer
// );
//only grow pid lookup if needed
if (batch.TargetNames.Count > targets.Length)
{
targets = new PID[batch.TargetNames.Count];
}
for (var i = 0; i < batch.TargetNames.Count; i++)
{
var pid = PID.FromAddress(_system.Address, batch.TargetNames[i]);
pid.Ref(_system);
targets[i] = pid;
}
var typeNames = batch.TypeNames.ToArray();
var m = _system.Metrics.Get <RemoteMetrics>().RemoteDeserializedMessageCount;
foreach (var envelope in batch.Envelopes)
{
var target = targets[envelope.Target];
if (envelope.RequestId != default)
{
target = target.WithRequestId(envelope.RequestId);
}
var typeName = typeNames[envelope.TypeId];
if (!_system.Metrics.IsNoop)
{
m.Inc(new[] { _system.Id, _system.Address, typeName });
}
object message;
try
{
message =
_serialization.Deserialize(typeName, envelope.MessageData, envelope.SerializerId);
//translate from on-the-wire representation to in-process representation
//this only applies to root level messages, and never on nested child messages
if (message is IRootSerialized serialized)
{
message = serialized.Deserialize(_system);
}
}
catch (Exception)
{
Logger.Log(_deserializationErrorLogLevel, "[EndpointReader] Unable to deserialize message with {Type} from {Remote}",
typeName, context.Peer
);
continue;
}
switch (message)
{
case Terminated msg:
Terminated(msg, target);
break;
case SystemMessage sys:
SystemMessage(sys, target);
break;
default:
ReceiveMessages(envelope, message, target);
break;
}
}
}
_system.Metrics.Get <RemoteMetrics>().RemoteEndpointDisconnectedCount.Inc(new[] { _system.Id, _system.Address, context.Peer });
Logger.LogDebug("[EndpointReader] Stream closed by {Remote}", context.Peer);
}
// ReSharper disable once SuggestBaseTypeForParameter
private PID Activate(Member activator, ClusterIdentity identity) =>
PID.FromAddress(activator.Address, $"placement-activator/{identity}${NextId()}");
