private async Task StartedAsync() { _logger.LogDebug($"Connecting to address {_address}"); _channel = new Channel(_address, _channelCredentials, _channelOptions); _client = new Remoting.RemotingClient(_channel); try { var res = await _client.ConnectAsync(new ConnectRequest()); _serializerId = res.DefaultSerializerId; _stream = _client.Receive(_callOptions); _streamWriter = _stream.RequestStream; } catch (Exception ex) { _logger.LogError($"GRPC Failed to connect to address {_address}\n{ex}"); //Wait for 2 seconds to restart and retry //Replace with Exponential Backoff await Task.Delay(2000); throw; } var _ = Task.Factory.StartNew(async() => { try { await _stream.ResponseStream.ForEachAsync(i => Actor.Done).ConfigureAwait(false); } catch (Exception x) { _logger.LogError($"Lost connection to address {_address}, reason {x.Message}"); var terminated = new EndpointTerminatedEvent { Address = _address }; Actor.EventStream.Publish(terminated); } }); var connected = new EndpointConnectedEvent { Address = _address }; Actor.EventStream.Publish(connected); _logger.LogDebug($"Connected to address {_address}"); }
private async Task StartedAsync() { _logger.LogDebug($"Connecting to address {_address}"); _channel = new Channel(_address, _channelCredentials, _channelOptions); _client = new Remoting.RemotingClient(_channel); var res = await _client.ConnectAsync(new ConnectRequest()); _serializerId = res.DefaultSerializerId; _stream = _client.Receive(_callOptions); var _ = Task.Factory.StartNew(async() => { try { await _stream.ResponseStream.ForEachAsync(i => Actor.Done); } catch (Exception x) { _logger.LogError($"Lost connection to address {_address}, reason {x.Message}"); var terminated = new EndpointTerminatedEvent { Address = _address }; Actor.EventStream.Publish(terminated); } }); var connected = new EndpointConnectedEvent { Address = _address }; Actor.EventStream.Publish(connected); _streamWriter = _stream.RequestStream; _logger.LogDebug($"Connected to address {_address}"); }
private static void OnEndpointConnected(EndpointConnectedEvent msg) { var endpoint = EnsureConnected(msg.Address); endpoint.Watcher.Tell(msg); }
private async Task RunAsync() { object?m = null; try { // Logger.LogDebug( // "[EndpointWriterMailbox] Running Mailbox Loop HasSystemMessages: {HasSystemMessages} HasUserMessages: {HasUserMessages} Suspended: {Suspended}", // _systemMessages.HasMessages, _userMessages.HasMessages, _suspended // ); var _ = _dispatcher !.Throughput; //not used for batch mailbox var batch = new List <RemoteDeliver>(_batchSize); var sys = _systemMessages.Pop(); if (sys is not null) { Logger.LogDebug("[EndpointWriterMailbox] Processing System Message {@Message}", sys); _suspended = sys switch { SuspendMailbox _ => true, EndpointConnectedEvent _ => false, _ => _suspended }; m = sys; switch (m) { case EndpointErrorEvent e: if (!_suspended) // Since it's already stopped, there is no need to throw the error { await _invoker !.InvokeUserMessageAsync(sys); } break; case EndpointConnectedEvent: //endpoint connected event is not a real system message, do not pass it to the invoker break; default: await _invoker !.InvokeSystemMessageAsync(sys); break; } if (sys is Stop) { // Logger.LogWarning("Endpoint writer is stopping..."); //Dump messages from user messages queue to deadletter and inform watchers about termination object?usrMsg; var droppedRemoteDeliverCount = 0; var remoteTerminateCount = 0; while ((usrMsg = _userMessages.Pop()) is not null) { switch (usrMsg) { case RemoteWatch msg: remoteTerminateCount++; msg.Watcher.SendSystemMessage(_system, new Terminated { Why = TerminatedReason.AddressTerminated, Who = msg.Watchee } ); break; case RemoteDeliver rd: droppedRemoteDeliverCount++; if (rd.Sender != null) { _system.Root.Send(rd.Sender, new DeadLetterResponse { Target = rd.Target }); } _system.EventStream.Publish(new DeadLetterEvent(rd.Target, rd.Message, rd.Sender)); break; } } if (droppedRemoteDeliverCount > 0) { Logger.LogInformation("[EndpointWriterMailbox] Dropped {count} user Messages for {Address}", droppedRemoteDeliverCount, _address ); } if (remoteTerminateCount > 0) { Logger.LogInformation("[EndpointWriterMailbox] Sent {Count} remote terminations for {Address}", remoteTerminateCount, _address ); } } } if (!_suspended) { batch.Clear(); object?msg; while ((msg = _userMessages.Pop()) is not null) { // Logger.LogDebug("[EndpointWriterMailbox] Processing User Message {@Message}", msg); switch (msg) { case RemoteWatch _: case RemoteUnwatch _: case RemoteTerminate _: await _invoker !.InvokeUserMessageAsync(msg); continue; } batch.Add((RemoteDeliver)msg); if (batch.Count >= _batchSize) { break; } } if (batch.Count > 0) { m = batch; // Logger.LogDebug("[EndpointWriterMailbox] Calling message invoker"); await _invoker !.InvokeUserMessageAsync(batch); } } } catch (Exception x) { // if (x is RpcException rpc && rpc.Status.StatusCode == StatusCode.Unavailable) // { // Logger.LogError( "Endpoint writer failed, status unavailable"); // } // else // { // Logger.LogError(x, "Endpoint writer failed"); // } _suspended = true; _invoker !.EscalateFailure(x, m); } Interlocked.Exchange(ref _status, MailboxStatus.Idle); if (_systemMessages.HasMessages || _userMessages.HasMessages & !_suspended) { Schedule(); } }
private static void OnEndpointConnected(EndpointConnectedEvent msg) { var endpoint = EnsureConnected(msg.Address); RootContext.Empty.Send(endpoint.Watcher, msg); }