private async Task StartedAsync()
{
_logger.LogDebug($"Connecting to address {_address}");
_channel = new Channel(_address, _channelCredentials, _channelOptions);
_client = new Remoting.RemotingClient(_channel);
try
{
var res = await _client.ConnectAsync(new ConnectRequest());
_serializerId = res.DefaultSerializerId;
_stream = _client.Receive(_callOptions);
_streamWriter = _stream.RequestStream;
}
catch (Exception ex)
{
_logger.LogError($"GRPC Failed to connect to address {_address}\n{ex}");
//Wait for 2 seconds to restart and retry
//Replace with Exponential Backoff
await Task.Delay(2000);
throw;
}
var _ = Task.Factory.StartNew(async() =>
{
try
{
await _stream.ResponseStream.ForEachAsync(i => Actor.Done).ConfigureAwait(false);
}
catch (Exception x)
{
_logger.LogError($"Lost connection to address {_address}, reason {x.Message}");
var terminated = new EndpointTerminatedEvent
{
Address = _address
};
Actor.EventStream.Publish(terminated);
}
});
var connected = new EndpointConnectedEvent
{
Address = _address
};
Actor.EventStream.Publish(connected);
_logger.LogDebug($"Connected to address {_address}");
}
private async Task StartedAsync()
{
_logger.LogDebug($"Connecting to address {_address}");
_channel = new Channel(_address, _channelCredentials, _channelOptions);
_client = new Remoting.RemotingClient(_channel);
var res = await _client.ConnectAsync(new ConnectRequest());
_serializerId = res.DefaultSerializerId;
_stream = _client.Receive(_callOptions);
var _ = Task.Factory.StartNew(async() =>
{
try
{
await _stream.ResponseStream.ForEachAsync(i => Actor.Done);
}
catch (Exception x)
{
_logger.LogError($"Lost connection to address {_address}, reason {x.Message}");
var terminated = new EndpointTerminatedEvent
{
Address = _address
};
Actor.EventStream.Publish(terminated);
}
});
var connected = new EndpointConnectedEvent
{
Address = _address
};
Actor.EventStream.Publish(connected);
_streamWriter = _stream.RequestStream;
_logger.LogDebug($"Connected to address {_address}");
}
private static void OnEndpointConnected(EndpointConnectedEvent msg)
{
var endpoint = EnsureConnected(msg.Address);
endpoint.Watcher.Tell(msg);
}
private async Task RunAsync()
{
object?m = null;
try
{
// Logger.LogDebug(
// "[EndpointWriterMailbox] Running Mailbox Loop HasSystemMessages: {HasSystemMessages} HasUserMessages: {HasUserMessages} Suspended: {Suspended}",
// _systemMessages.HasMessages, _userMessages.HasMessages, _suspended
// );
var _ = _dispatcher !.Throughput; //not used for batch mailbox
var batch = new List <RemoteDeliver>(_batchSize);
var sys = _systemMessages.Pop();
if (sys is not null)
{
Logger.LogDebug("[EndpointWriterMailbox] Processing System Message {@Message}", sys);
_suspended = sys switch
{
SuspendMailbox _ => true,
EndpointConnectedEvent _ => false,
_ => _suspended
};
m = sys;
switch (m)
{
case EndpointErrorEvent e:
if (!_suspended) // Since it's already stopped, there is no need to throw the error
{
await _invoker !.InvokeUserMessageAsync(sys);
}
break;
case EndpointConnectedEvent:
//endpoint connected event is not a real system message, do not pass it to the invoker
break;
default:
await _invoker !.InvokeSystemMessageAsync(sys);
break;
}
if (sys is Stop)
{
// Logger.LogWarning("Endpoint writer is stopping...");
//Dump messages from user messages queue to deadletter and inform watchers about termination
object?usrMsg;
var droppedRemoteDeliverCount = 0;
var remoteTerminateCount = 0;
while ((usrMsg = _userMessages.Pop()) is not null)
{
switch (usrMsg)
{
case RemoteWatch msg:
remoteTerminateCount++;
msg.Watcher.SendSystemMessage(_system, new Terminated
{
Why = TerminatedReason.AddressTerminated,
Who = msg.Watchee
}
);
break;
case RemoteDeliver rd:
droppedRemoteDeliverCount++;
if (rd.Sender != null)
{
_system.Root.Send(rd.Sender, new DeadLetterResponse {
Target = rd.Target
});
}
_system.EventStream.Publish(new DeadLetterEvent(rd.Target, rd.Message, rd.Sender));
break;
}
}
if (droppedRemoteDeliverCount > 0)
{
Logger.LogInformation("[EndpointWriterMailbox] Dropped {count} user Messages for {Address}", droppedRemoteDeliverCount,
_address
);
}
if (remoteTerminateCount > 0)
{
Logger.LogInformation("[EndpointWriterMailbox] Sent {Count} remote terminations for {Address}", remoteTerminateCount,
_address
);
}
}
}
if (!_suspended)
{
batch.Clear();
object?msg;
while ((msg = _userMessages.Pop()) is not null)
{
// Logger.LogDebug("[EndpointWriterMailbox] Processing User Message {@Message}", msg);
switch (msg)
{
case RemoteWatch _:
case RemoteUnwatch _:
case RemoteTerminate _:
await _invoker !.InvokeUserMessageAsync(msg);
continue;
}
batch.Add((RemoteDeliver)msg);
if (batch.Count >= _batchSize)
{
break;
}
}
if (batch.Count > 0)
{
m = batch;
// Logger.LogDebug("[EndpointWriterMailbox] Calling message invoker");
await _invoker !.InvokeUserMessageAsync(batch);
}
}
}
catch (Exception x)
{
// if (x is RpcException rpc && rpc.Status.StatusCode == StatusCode.Unavailable)
// {
// Logger.LogError( "Endpoint writer failed, status unavailable");
// }
// else
// {
// Logger.LogError(x, "Endpoint writer failed");
// }
_suspended = true;
_invoker !.EscalateFailure(x, m);
}
Interlocked.Exchange(ref _status, MailboxStatus.Idle);
if (_systemMessages.HasMessages || _userMessages.HasMessages & !_suspended)
{
Schedule();
}
}
private static void OnEndpointConnected(EndpointConnectedEvent msg)
{
var endpoint = EnsureConnected(msg.Address);
RootContext.Empty.Send(endpoint.Watcher, msg);
}