예제 #1
0
 public SiloStatisticsManager(
     IOptions <SiloStatisticsOptions> statisticsOptions,
     IOptions <LoadSheddingOptions> loadSheddingOptions,
     IOptions <StorageOptions> azureStorageOptions,
     ILocalSiloDetails siloDetails,
     SerializationManager serializationManager,
     ITelemetryProducer telemetryProducer,
     IHostEnvironmentStatistics hostEnvironmentStatistics,
     IAppEnvironmentStatistics appEnvironmentStatistics,
     ILoggerFactory loggerFactory,
     IOptions <SiloMessagingOptions> messagingOptions)
 {
     this.siloDetails    = siloDetails;
     this.storageOptions = azureStorageOptions.Value;
     MessagingStatisticsGroup.Init(true);
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init(true);
     ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
     SchedulerStatisticsGroup.Init(loggerFactory);
     StorageStatisticsGroup.Init();
     TransactionsStatisticsGroup.Init();
     this.logger = loggerFactory.CreateLogger <SiloStatisticsManager>();
     this.hostEnvironmentStatistics = hostEnvironmentStatistics;
     this.logStatistics             = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
     this.MetricsTable      = new SiloPerformanceMetrics(this.hostEnvironmentStatistics, appEnvironmentStatistics, loggerFactory, loadSheddingOptions);
     this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
 }
예제 #2
0
 internal void RecordOpenedConnection(GatewayInboundConnection connection, GrainId clientId)
 {
     lock (lockable)
     {
         logger.LogInformation((int)ErrorCode.GatewayClientOpenedSocket, "Recorded opened connection from endpoint {EndPoint}, client ID {ClientId}.", connection.RemoteEndpoint, clientId);
         ClientState clientState;
         if (clients.TryGetValue(clientId, out clientState))
         {
             var oldSocket = clientState.Connection;
             if (oldSocket != null)
             {
                 // The old socket will be closed by itself later.
                 clientConnections.TryRemove(oldSocket, out _);
             }
         }
         else
         {
             clientState       = new ClientState(clientId, messagingOptions.ClientDropTimeout);
             clients[clientId] = clientState;
             MessagingStatisticsGroup.ConnectedClientCount.Increment();
         }
         clientState.RecordConnection(connection);
         clientConnections[connection] = clientState;
         clientRegistrar.ClientAdded(clientId);
         NetworkingStatisticsGroup.OnOpenedGatewayDuplexSocket();
     }
 }
 protected virtual void RecordClosedSocket(Socket sock)
 {
     if (TryRemoveClosedSocket(sock))
     {
         NetworkingStatisticsGroup.OnClosedReceivingSocket();
     }
 }
예제 #4
0
파일: Gateway.cs 프로젝트: wanglong/orleans
 internal void RecordOpenedSocket(Socket sock, GrainId clientId)
 {
     lock (lockable)
     {
         logger.Info(ErrorCode.GatewayClientOpenedSocket, "Recorded opened socket from endpoint {0}, client ID {1}.", sock.RemoteEndPoint, clientId);
         ClientState clientState;
         if (clients.TryGetValue(clientId, out clientState))
         {
             var oldSocket = clientState.Socket;
             if (oldSocket != null)
             {
                 // The old socket will be closed by itself later.
                 ClientState ignore;
                 clientSockets.TryRemove(oldSocket, out ignore);
             }
             QueueRequest(clientState, null);
         }
         else
         {
             int gatewayToUse = nextGatewaySenderToUseForRoundRobin % senders.Length;
             nextGatewaySenderToUseForRoundRobin++; // under Gateway lock
             clientState       = new ClientState(clientId, gatewayToUse);
             clients[clientId] = clientState;
             MessagingStatisticsGroup.ConnectedClientCount.Increment();
         }
         clientState.RecordConnection(sock);
         clientSockets[sock] = clientState;
         clientRegistrar.ClientAdded(clientId);
         NetworkingStatisticsGroup.OnOpenedGatewayDuplexSocket();
     }
 }
예제 #5
0
        protected virtual bool RecordOpenedSocket(Socket sock)
        {
            GrainId client;
            if (!ReceiveSocketPreample(sock, false, out client)) return false;

            NetworkingStatisticsGroup.OnOpenedReceiveSocket();
            return true;
        }
예제 #6
0
        private static Task OnConnectedAsync(ConnectionContext context)
        {
            var connection = context.Features.Get <Connection>();

            context.ConnectionClosed.Register(OnConnectionClosedDelegate, connection);

            NetworkingStatisticsGroup.OnOpenedSocket(connection.ConnectionDirection);
            return(connection.RunInternal());
        }
 private void CloseSocket(Socket socket)
 {
     SocketManager.CloseSocket(socket);
     NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
     if (Interlocked.Decrement(ref this.connectedCount) == 0)
     {
         MsgCenter.OnGatewayConnectionClosed();
     }
 }
예제 #8
0
파일: Connection.cs 프로젝트: wjire/orleans
        private void CloseInternal(Exception exception)
        {
            if (!this.IsValid)
            {
                return;
            }

            lock (this.lockObj)
            {
                try
                {
                    if (!this.IsValid)
                    {
                        return;
                    }
                    this.IsValid = false;
                    NetworkingStatisticsGroup.OnClosedSocket(this.ConnectionDirection);

                    this.closeRegistration.Dispose();
                    this.closeRegistration = default;

                    if (this.Log.IsEnabled(LogLevel.Information))
                    {
                        this.Log.LogInformation(
                            "Closing connection with remote endpoint {EndPoint}",
                            this.RemoteEndPoint,
                            Environment.StackTrace);
                    }

                    // Try to gracefully stop the reader/writer loops.
                    this.Context.Transport.Input.CancelPendingRead();
                    this.Context.Transport.Output.CancelPendingFlush();
                    this.outgoingMessageWriter.TryComplete();

                    if (exception is null)
                    {
                        this.Context.Abort();
                    }
                    else
                    {
                        var abortedException = exception as ConnectionAbortedException
                                               ?? new ConnectionAbortedException(
                            $"Connection closed. See {nameof(Exception.InnerException)}",
                            exception);

                        this.Context.Abort(abortedException);
                    }
                }
                catch (Exception innerException)
                {
                    // Swallow any exceptions here.
                    this.Log.LogWarning(innerException, "Exception closing connection with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, innerException);
                }
            }
        }
예제 #9
0
파일: Gateway.cs 프로젝트: wanglong/orleans
            internal void RecordDisconnection()
            {
                if (Socket == null)
                {
                    return;
                }

                DisconnectedSince = DateTime.UtcNow;
                Socket            = null;
                NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
            }
예제 #10
0
 public SiloStatisticsManager(
     IOptions <StatisticsOptions> statisticsOptions,
     ITelemetryProducer telemetryProducer,
     ILoggerFactory loggerFactory)
 {
     MessagingStatisticsGroup.Init();
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init();
     StorageStatisticsGroup.Init();
     this.logStatistics     = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, loggerFactory);
     this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
 }
예제 #11
0
        /* Temp fn only for testing Nekara service */
        private static System.Threading.Tasks.Task OnConnectedAsync_temp(ConnectionContext context)
        {
            var connection = context.Features.Get <Connection>();

            context.ConnectionClosed.Register(OnConnectionClosedDelegate, connection);

            NetworkingStatisticsGroup.OnOpenedSocket(connection.ConnectionDirection);

            var _t1 = connection.RunInternal().InnerTask;

            return(_t1);
        }
예제 #12
0
        protected virtual bool RecordOpenedSocket(Socket sock)
        {
            Guid client;

            if (!ReceiveSocketPreample(sock, false, out client))
            {
                return(false);
            }

            NetworkingStatisticsGroup.OnOpenedReceiveSocket();
            return(true);
        }
예제 #13
0
 internal SiloStatisticsManager(GlobalConfiguration globalConfig, NodeConfiguration nodeConfig)
 {
     MessagingStatisticsGroup.Init(true);
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init(true);
     ApplicationRequestsStatisticsGroup.Init(globalConfig.ResponseTimeout);
     SchedulerStatisticsGroup.Init();
     StorageStatisticsGroup.Init();
     runtimeStats          = new RuntimeStatisticsGroup();
     logStatistics         = new LogStatistics(nodeConfig.StatisticsLogWriteInterval, true);
     MetricsTable          = new SiloPerformanceMetrics(runtimeStats, nodeConfig);
     perfCountersPublisher = new PerfCountersStatistics(nodeConfig.StatisticsPerfCountersWriteInterval);
 }
예제 #14
0
 public SiloStatisticsManager(NodeConfiguration nodeConfiguration, ILocalSiloDetails siloDetails, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory, IOptions <MessagingOptions> messagingOptions)
 {
     this.siloDetails = siloDetails;
     MessagingStatisticsGroup.Init(true);
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init(true);
     ApplicationRequestsStatisticsGroup.Init(messagingOptions.Value.ResponseTimeout);
     SchedulerStatisticsGroup.Init(loggerFactory);
     StorageStatisticsGroup.Init();
     TransactionsStatisticsGroup.Init();
     this.logger            = loggerFactory.CreateLogger <SiloStatisticsManager>();
     runtimeStats           = new RuntimeStatisticsGroup(loggerFactory);
     this.logStatistics     = new LogStatistics(nodeConfiguration.StatisticsLogWriteInterval, true, serializationManager, loggerFactory);
     this.MetricsTable      = new SiloPerformanceMetrics(this.runtimeStats, loggerFactory, nodeConfiguration);
     this.countersPublisher = new CountersStatistics(nodeConfiguration.StatisticsPerfCountersWriteInterval, telemetryProducer, loggerFactory);
 }
예제 #15
0
 public SiloStatisticsManager(
     IOptions <SiloStatisticsOptions> statisticsOptions,
     SerializationManager serializationManager,
     ITelemetryProducer telemetryProducer,
     ILoggerFactory loggerFactory)
 {
     MessagingStatisticsGroup.Init(true);
     MessagingProcessingStatisticsGroup.Init();
     NetworkingStatisticsGroup.Init(true);
     ApplicationRequestsStatisticsGroup.Init();
     SchedulerStatisticsGroup.Init(loggerFactory);
     StorageStatisticsGroup.Init();
     TransactionsStatisticsGroup.Init();
     this.logStatistics     = new LogStatistics(statisticsOptions.Value.LogWriteInterval, true, serializationManager, loggerFactory);
     this.countersPublisher = new CountersStatistics(statisticsOptions.Value.PerfCountersWriteInterval, telemetryProducer, loggerFactory);
 }
        public SiloStatisticsManager(SiloInitializationParameters initializationParams)
        {
            MessagingStatisticsGroup.Init(true);
            MessagingProcessingStatisticsGroup.Init();
            NetworkingStatisticsGroup.Init(true);
            ApplicationRequestsStatisticsGroup.Init(initializationParams.GlobalConfig.ResponseTimeout);
            SchedulerStatisticsGroup.Init();
            StorageStatisticsGroup.Init();
            runtimeStats           = new RuntimeStatisticsGroup();
            this.logStatistics     = new LogStatistics(initializationParams.NodeConfig.StatisticsLogWriteInterval, true);
            this.MetricsTable      = new SiloPerformanceMetrics(this.runtimeStats, initializationParams.NodeConfig);
            this.countersPublisher = new CountersStatistics(initializationParams.NodeConfig.StatisticsPerfCountersWriteInterval);

            initializationParams.ClusterConfig.OnConfigChange(
                "Defaults/LoadShedding",
                () => this.MetricsTable.NodeConfig = initializationParams.NodeConfig,
                false);
        }
예제 #17
0
        public SiloStatisticsManager(SiloInitializationParameters initializationParams, SerializationManager serializationManager, ITelemetryProducer telemetryProducer, ILoggerFactory loggerFactory)
        {
            MessagingStatisticsGroup.Init(true);
            MessagingProcessingStatisticsGroup.Init();
            NetworkingStatisticsGroup.Init(true);
            ApplicationRequestsStatisticsGroup.Init(initializationParams.ClusterConfig.Globals.ResponseTimeout);
            SchedulerStatisticsGroup.Init(loggerFactory);
            StorageStatisticsGroup.Init();
            TransactionsStatisticsGroup.Init();
            this.logger            = new LoggerWrapper <SiloStatisticsManager>(loggerFactory);
            runtimeStats           = new RuntimeStatisticsGroup(loggerFactory);
            this.logStatistics     = new LogStatistics(initializationParams.NodeConfig.StatisticsLogWriteInterval, true, serializationManager, loggerFactory);
            this.MetricsTable      = new SiloPerformanceMetrics(this.runtimeStats, loggerFactory, initializationParams.NodeConfig);
            this.countersPublisher = new CountersStatistics(initializationParams.NodeConfig.StatisticsPerfCountersWriteInterval, telemetryProducer, loggerFactory);

            initializationParams.ClusterConfig.OnConfigChange(
                "Defaults/LoadShedding",
                () => this.MetricsTable.NodeConfig = initializationParams.NodeConfig,
                false);
        }
예제 #18
0
        public override void Stop()
        {
            IsLive = false;
            receiver.Stop();
            base.Stop();
            Socket s;

            lock (Lockable)
            {
                s      = Socket;
                Socket = null;
            }
            if (s == null)
            {
                return;
            }

            SocketManager.CloseSocket(s);
            NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
        }
예제 #19
0
        public override void Stop()
        {
            IsLive = false;
            receiver.Stop();
            base.Stop();
            RuntimeClient.Current.BreakOutstandingMessagesToDeadSilo(Silo);
            Socket s;

            lock (Lockable)
            {
                s      = Socket;
                Socket = null;
            }
            if (s == null)
            {
                return;
            }

            SocketManager.CloseSocket(s);
            NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
        }
예제 #20
0
        // passed the exact same socket on which it got SocketException. This way we prevent races between connect and disconnect.
        public void MarkAsDisconnected(Socket socket2Disconnect)
        {
            Socket s = null;

            lock (Lockable)
            {
                if (socket2Disconnect == null || Socket == null)
                {
                    return;
                }

                if (Socket == socket2Disconnect)  // handles races between connect and disconnect, since sometimes we grab the socket outside lock.
                {
                    s      = Socket;
                    Socket = null;
                    Log.Warn(ErrorCode.ProxyClient_MarkGatewayDisconnected, String.Format("Marking gateway at address {0} as Disconnected", Address));
                    if (MsgCenter != null && MsgCenter.GatewayManager != null)
                    {
                        // We need a refresh...
                        MsgCenter.GatewayManager.ExpediteUpdateLiveGatewaysSnapshot();
                    }
                }
            }
            if (s != null)
            {
                SocketManager.CloseSocket(s);
                NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
            }
            if (socket2Disconnect == s)
            {
                return;
            }

            SocketManager.CloseSocket(socket2Disconnect);
            NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
        }
예제 #21
0
 private void CloseSocket(Socket socket)
 {
     SocketManager.CloseSocket(socket);
     NetworkingStatisticsGroup.OnClosedGatewayDuplexSocket();
     MsgCenter.OnGatewayConnectionClosed();
 }
예제 #22
0
        public void Connect()
        {
            if (!MsgCenter.Running)
            {
                if (Log.IsVerbose)
                {
                    Log.Verbose(ErrorCode.ProxyClient_MsgCtrNotRunning, "Ignoring connection attempt to gateway {0} because the proxy message center is not running", Address);
                }
                return;
            }

            // Yes, we take the lock around a Sleep. The point is to ensure that no more than one thread can try this at a time.
            // There's still a minor problem as written -- if the sending thread and receiving thread both get here, the first one
            // will try to reconnect. eventually do so, and then the other will try to reconnect even though it doesn't have to...
            // Hopefully the initial "if" statement will prevent that.
            lock (Lockable)
            {
                if (!IsLive)
                {
                    if (Log.IsVerbose)
                    {
                        Log.Verbose(ErrorCode.ProxyClient_DeadGateway, "Ignoring connection attempt to gateway {0} because this gateway connection is already marked as non live", Address);
                    }
                    return; // if the connection is already marked as dead, don't try to reconnect. It has been doomed.
                }

                for (var i = 0; i < ProxiedMessageCenter.CONNECT_RETRY_COUNT; i++)
                {
                    try
                    {
                        if (Socket != null)
                        {
                            if (Socket.Connected)
                            {
                                return;
                            }

                            MarkAsDisconnected(Socket); // clean up the socket before reconnecting.
                        }
                        if (lastConnect != new DateTime())
                        {
                            // We already tried at least once in the past to connect to this GW.
                            // If we are no longer connected to this GW and it is no longer in the list returned
                            // from the GatewayProvider, consider directly this connection dead.
                            if (!MsgCenter.GatewayManager.GetLiveGateways().Contains(Address))
                            {
                                break;
                            }

                            // Wait at least ProxiedMessageCenter.MINIMUM_INTERCONNECT_DELAY before reconnection tries
                            var millisecondsSinceLastAttempt = DateTime.UtcNow - lastConnect;
                            if (millisecondsSinceLastAttempt < ProxiedMessageCenter.MINIMUM_INTERCONNECT_DELAY)
                            {
                                var wait = ProxiedMessageCenter.MINIMUM_INTERCONNECT_DELAY - millisecondsSinceLastAttempt;
                                if (Log.IsVerbose)
                                {
                                    Log.Verbose(ErrorCode.ProxyClient_PauseBeforeRetry, "Pausing for {0} before trying to connect to gateway {1} on trial {2}", wait, Address, i);
                                }
                                Thread.Sleep(wait);
                            }
                        }
                        lastConnect = DateTime.UtcNow;
                        Socket      = new Socket(Silo.Endpoint.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
                        SocketManager.Connect(Socket, Silo.Endpoint, MsgCenter.MessagingConfiguration.OpenConnectionTimeout);
                        NetworkingStatisticsGroup.OnOpenedGatewayDuplexSocket();
                        MsgCenter.OnGatewayConnectionOpen();
                        SocketManager.WriteConnectionPreamble(Socket, MsgCenter.ClientId);  // Identifies this client
                        Log.Info(ErrorCode.ProxyClient_Connected, "Connected to gateway at address {0} on trial {1}.", Address, i);
                        return;
                    }
                    catch (Exception ex)
                    {
                        Log.Warn(ErrorCode.ProxyClient_CannotConnect, $"Unable to connect to gateway at address {Address} on trial {i} (Exception: {ex.Message})");
                        MarkAsDisconnected(Socket);
                    }
                }
                // Failed too many times -- give up
                MarkAsDead();
            }
        }
예제 #23
0
        private void CloseInternal(Exception exception)
        {
            if (!this.IsValid)
            {
                return;
            }

            lock (this.lockObj)
            {
                try
                {
                    if (!this.IsValid)
                    {
                        return;
                    }
                    this.IsValid = false;
                    NetworkingStatisticsGroup.OnClosedSocket(this.ConnectionDirection);

                    if (this.Log.IsEnabled(LogLevel.Information))
                    {
                        if (exception is null)
                        {
                            this.Log.LogInformation(
                                "Closing connection with remote endpoint {EndPoint}",
                                this.RemoteEndPoint);
                        }
                        else
                        {
                            this.Log.LogInformation(
                                exception,
                                "Closing connection with remote endpoint {EndPoint}. Exception: {Exception}",
                                this.RemoteEndPoint,
                                exception);
                        }
                    }

                    // Try to gracefully stop the reader/writer loops, if they are running.
                    try
                    {
                        if (_processIncomingTask is Task task && !task.IsCompleted)
                        {
                            this.Context.Transport.Input.CancelPendingRead();
                        }
                    }
                    catch (Exception cancelException)
                    {
                        // Swallow any exceptions here.
                        this.Log.LogWarning(cancelException, "Exception canceling pending read with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, cancelException);
                    }

                    try
                    {
                        if (_processOutgoingTask is Task task && !task.IsCompleted)
                        {
                            this.Context.Transport.Output.CancelPendingFlush();
                        }
                    }
                    catch (Exception cancelException)
                    {
                        // Swallow any exceptions here.
                        this.Log.LogWarning(cancelException, "Exception canceling pending flush with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, cancelException);
                    }

                    this.outgoingMessageWriter.TryComplete();

                    if (exception is null)
                    {
                        this.Context.Abort();
                    }
                    else
                    {
                        var abortedException = exception as ConnectionAbortedException
                                               ?? new ConnectionAbortedException(
                            $"Connection closed. See {nameof(Exception.InnerException)}",
                            exception);

                        this.Context.Abort(abortedException);
                    }
                }
                catch (Exception innerException)
                {
                    // Swallow any exceptions here.
                    this.Log.LogWarning(innerException, "Exception closing connection with remote endpoint {EndPoint}: {Exception}", this.RemoteEndPoint, innerException);
                }
            }
        }
예제 #24
0
        /// <summary>
        /// Close the connection. This method should only be called by <see cref="StartClosing(Exception)"/>.
        /// </summary>
        private async Task CloseAsync()
        {
            NetworkingStatisticsGroup.OnClosedSocket(this.ConnectionDirection);

            // Signal the outgoing message processor to exit gracefully.
            this.outgoingMessageWriter.TryComplete();

            var transportFeature = Context.Features.Get <IUnderlyingTransportFeature>();
            var transport        = transportFeature?.Transport ?? _transport;

            transport.Input.CancelPendingRead();
            transport.Output.CancelPendingFlush();

            // Try to gracefully stop the reader/writer loops, if they are running.
            if (_processIncomingTask is { IsCompleted : false } incoming)
            {
                try
                {
                    await incoming;
                }
                catch (Exception processIncomingException)
                {
                    // Swallow any exceptions here.
                    this.Log.LogWarning(processIncomingException, "Exception processing incoming messages on connection {Connection}", this);
                }
            }

            if (_processOutgoingTask is { IsCompleted : false } outgoing)
            {
                try
                {
                    await outgoing;
                }
                catch (Exception processOutgoingException)
                {
                    // Swallow any exceptions here.
                    this.Log.LogWarning(processOutgoingException, "Exception processing outgoing messages on connection {Connection}", this);
                }
            }

            // Only wait for the transport to close if the connection actually started being processed.
            if (_processIncomingTask is not null && _processOutgoingTask is not null)
            {
                // Wait for the transport to signal that it's closed before disposing it.
                await _transportConnectionClosed.Task;
            }

            try
            {
                await this.Context.DisposeAsync();
            }
            catch (Exception abortException)
            {
                // Swallow any exceptions here.
                this.Log.LogWarning(abortException, "Exception terminating connection {Connection}", this);
            }

            // Reject in-flight messages.
            foreach (var message in this.inflight)
            {
                this.OnSendMessageFailure(message, "Connection terminated");
            }

            this.inflight.Clear();

            // Reroute enqueued messages.
            var i = 0;

            while (this.outgoingMessages.Reader.TryRead(out var message))
            {
                if (i == 0 && Log.IsEnabled(LogLevel.Information))
                {
                    this.Log.LogInformation(
                        "Rerouting messages for remote endpoint {EndPoint}",
                        this.RemoteEndPoint?.ToString() ?? "(never connected)");
                }

                ++i;
                this.RetryMessage(message);
            }

            if (i > 0 && this.Log.IsEnabled(LogLevel.Information))
            {
                this.Log.LogInformation(
                    "Rerouted {Count} messages for remote endpoint {EndPoint}",
                    i,
                    this.RemoteEndPoint?.ToString() ?? "(never connected)");
            }
        }