/// <summary>
        ///     Handles the event for a new server joining the cluster.
        /// </summary>
        /// <param name="id">The id of the server joining.</param>
        /// <param name="remoteServer">The server joining.</param>
        protected void HandleServerJoinEvent(ushort id, IRemoteServer remoteServer)
        {
            EventHandler <ServerJoinedEventArgs> handler = ServerJoined;

            if (handler != null)
            {
                void DoServerJoinEvent()
                {
                    long startTimestamp = Stopwatch.GetTimestamp();

                    try
                    {
                        handler?.Invoke(this, new ServerJoinedEventArgs(remoteServer, id, this));
                    }
                    catch (Exception e)
                    {
                        serverJoinedEventFailuresCounter.Increment();

                        // TODO this seems bad, shoudln't we disconnect them?
                        logger.Error("A plugin encountered an error whilst handling the ServerJoined event. The server will still be connected. (See logs for exception)", e);
                    }

                    double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;

                    serverJoinedEventTimeHistogram.Report(time);
                }

                threadHelper.DispatchIfNeeded(DoServerJoinEvent);
            }
        }
        /// <summary>
        ///     Handles the event for a server leaving the cluster.
        /// </summary>
        /// <param name="id">The server leaving.</param>
        /// <param name="remoteServer">The server leaving.</param>
        protected void HandleServerLeaveEvent(ushort id, IRemoteServer remoteServer)
        {
            EventHandler <ServerLeftEventArgs> handler = ServerLeft;

            if (handler != null)
            {
                void DoServerLeaveEvent()
                {
                    long startTimestamp = Stopwatch.GetTimestamp();

                    try
                    {
                        handler?.Invoke(this, new ServerLeftEventArgs(remoteServer, id, this));
                    }
                    catch (Exception e)
                    {
                        serverLeftEventFailuresCounter.Increment();

                        logger.Error("A plugin encountered an error whilst handling the ServerLeft event. (See logs for exception)", e);
                    }

                    double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;

                    serverLeftEventTimeHistogram.Report(time);
                }

                threadHelper.DispatchIfNeeded(DoServerLeaveEvent);
            }
        }
        /// <summary>
        /// Called when the connection is lost.
        /// </summary>
        /// <param name="error">The socket error that ocurred</param>
        /// <param name="exception">The exception that ocurred.</param>
        private void DisconnectedHandler(SocketError error, Exception exception)
        {
            serverGroup.DisconnectedHandler(connection, this, exception);

            EventHandler <ServerDisconnectedEventArgs> handler = ServerDisconnected;

            if (handler != null)
            {
                void DoServerDisconnectedEvent()
                {
                    long startTimestamp = Stopwatch.GetTimestamp();

                    try
                    {
                        handler?.Invoke(this, new ServerDisconnectedEventArgs(this, error, exception));
                    }
                    catch (Exception e)
                    {
                        serverDisconnectedEventFailuresCounter.Increment();

                        logger.Error("A plugin encountered an error whilst handling the ServerDisconnected event. (See logs for exception)", e);
                    }

                    double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;

                    serverDisconnectedEventTimeHistogram.Report(time);
                }

                threadHelper.DispatchIfNeeded(DoServerDisconnectedEvent);
            }
        }
        internal void Connect()
        {
            IEnumerable <IPAddress> addresses = Dns.GetHostEntry(Host).AddressList;

            // Try to connect to an IP address
            // TODO this might not reconnect to the same IP, break out option to prioritised last connected to.
            // TODO this will always try the same IP address, break out round robin option for load balancing
            this.connection = GetResultOfFirstSuccessfulInvocationOf(addresses, (address) =>
            {
                NetworkClientConnection c = serverGroup.GetConnection(address, Port);

                c.MessageReceived += MessageReceivedHandler;
                c.Disconnected    += DisconnectedHandler;

                c.Connect();

                return(c);
            });

            using (DarkRiftWriter writer = DarkRiftWriter.Create())
            {
                writer.Write(remoteServerManager.ServerID);

                using (Message message = Message.Create((ushort)CommandCode.Identify, writer))
                {
                    message.IsCommandMessage = true;
                    SendMessage(message, SendMode.Reliable);
                }
            }

            EventHandler <ServerConnectedEventArgs> handler = ServerConnected;

            if (handler != null)
            {
                void DoServerConnectedEvent()
                {
                    long startTimestamp = Stopwatch.GetTimestamp();

                    try
                    {
                        handler?.Invoke(this, new ServerConnectedEventArgs(this));
                    }
                    catch (Exception e)
                    {
                        serverConnectedEventFailuresCounter.Increment();

                        // TODO this seems bad, shouldn't we disconenct them?
                        logger.Error("A plugin encountered an error whilst handling the ServerConnected event. The server will still be connected. (See logs for exception)", e);
                    }

                    double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;

                    serverConnectedEventTimeHistogram.Report(time);
                }

                threadHelper.DispatchIfNeeded(DoServerConnectedEvent);
            }
        }
        /// <summary>
        ///     Sends a message to the server.
        /// </summary>
        /// <param name="message">The message to send.</param>
        /// <param name="sendMode">How the message should be sent.</param>
        /// <returns>Whether the send was successful.</returns>
        public bool SendMessage(Message message, SendMode sendMode)
        {
            bool success = connection?.SendMessage(message.ToBuffer(), sendMode) ?? false;

            if (success)
            {
                messagesSentCounter.Increment();
            }

            return(success);
        }
        /// <summary>
        ///     Callback for when data is received.
        /// </summary>
        /// <param name="buffer">The data recevied.</param>
        /// <param name="sendMode">The SendMode used to send the data.</param>
        private void MessageReceivedHandler(MessageBuffer buffer, SendMode sendMode)
        {
            messagesReceivedCounter.Increment();

            using (Message message = Message.Create(buffer, true))
            {
                if (message.IsCommandMessage)
                {
                    logger.Warning($"Server {ID} sent us a command message unexpectedly. This server may be configured to expect clients to connect.");
                }

                HandleMessage(message, sendMode);
            }
        }
        /// <summary>
        ///     Sets the connection being used by this remote server.
        /// </summary>
        /// <param name="pendingServer">The connection to switch to.</param>
        internal void SetConnection(PendingDownstreamRemoteServer pendingServer)
        {
            if (connection != null)
            {
                connection.MessageReceived -= MessageReceivedHandler;
                connection.Disconnected    -= DisconnectedHandler;
            }

            connection = pendingServer.Connection;

            // Switch out message received handler from the pending server
            connection.MessageReceived = MessageReceivedHandler;
            connection.Disconnected    = DisconnectedHandler;

            EventHandler <ServerConnectedEventArgs> handler = ServerConnected;

            if (handler != null)
            {
                void DoServerConnectedEvent()
                {
                    long startTimestamp = Stopwatch.GetTimestamp();

                    try
                    {
                        handler?.Invoke(this, new ServerConnectedEventArgs(this));
                    }
                    catch (Exception e)
                    {
                        serverConnectedEventFailuresCounter.Increment();

                        logger.Error("A plugin encountered an error whilst handling the ServerConnected event. The server will still be connected. (See logs for exception)", e);
                    }

                    double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;

                    serverConnectedEventTimeHistogram.Report(time);
                }

                threadHelper.DispatchIfNeeded(DoServerConnectedEvent);
            }

            // Handle all messages that had queued
            foreach (PendingDownstreamRemoteServer.QueuedMessage queuedMessage in pendingServer.GetQueuedMessages())
            {
                HandleMessage(queuedMessage.Message, queuedMessage.SendMode);

                queuedMessage.Message.Dispose();
            }
        }
        /// <summary>
        ///     Handles a client disconnecting.
        /// </summary>
        /// <param name="client">The client disconnecting.</param>
        /// <param name="localDisconnect">If the disconnection was caused by a call to <see cref="Client.Disconnect"/></param>
        /// <param name="error">The error that caused the disconnect.</param>
        /// <param name="exception">The exception that caused the disconnect.</param>
        internal void HandleDisconnection(Client client, bool localDisconnect, SocketError error, Exception exception)
        {
            // If we're not in the current list of clients we've already disconnected
            if (!DeallocateID(client.ID, out int noClients))
            {
                return;
            }

            //Inform plugins of the disconnection
            EventHandler <ClientDisconnectedEventArgs> handler = ClientDisconnected;

            if (handler != null)
            {
                threadHelper.DispatchIfNeeded(
                    delegate()
                {
#if PRO
                    long startTimestamp = Stopwatch.GetTimestamp();
#endif
                    try
                    {
                        handler.Invoke(this, new ClientDisconnectedEventArgs(client, localDisconnect, error, exception));
                    }
                    catch (Exception e)
                    {
                        logger.Error("A plugin encountered an error whilst handling the ClientDisconnected event. (See logs for exception)", e);
#if PRO
                        clientDisconnectedEventFailuresCounter.Increment();
#endif
                        return;
                    }

#if PRO
                    double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;
                    clientDisconnectedEventTimeHistogram.Report(time);
#endif
                },
                    delegate(ActionDispatcherTask t)
                {
                    FinaliseClientDisconnect(exception, error, client, noClients);
                }
                    );
            }
            else
            {
                FinaliseClientDisconnect(exception, error, client, noClients);
            }
        }
        /// <summary>
        ///     Callback for when data is received.
        /// </summary>
        /// <param name="buffer">The data recevied.</param>
        /// <param name="sendMode">The SendMode used to send the data.</param>
        private void MessageReceivedHandler(MessageBuffer buffer, SendMode sendMode)
        {
            messagesReceivedCounter.Increment();

            using (Message message = Message.Create(buffer, true))
            {
                if (message.IsCommandMessage)
                {
                    HandleCommand(message);
                }
                else
                {
                    HandleMessage(message, sendMode);
                }
            }
        }
        /// <summary>
        ///     Handles a message received.
        /// </summary>
        /// <param name="message">The message that was received.</param>
        /// <param name="sendMode">The send mode the emssage was received with.</param>
        private void HandleMessage(Message message, SendMode sendMode)
        {
            // Get another reference to the message so 1. we can control the backing array's lifecycle and thus it won't get disposed of before we dispatch, and
            // 2. because the current message will be disposed of when this method returns.
            Message messageReference = message.Clone();

            void DoMessageReceived()
            {
                ServerMessageReceivedEventArgs args = ServerMessageReceivedEventArgs.Create(message, sendMode, this);

                long startTimestamp = Stopwatch.GetTimestamp();

                try
                {
                    MessageReceived?.Invoke(this, args);
                }
                catch (Exception e)
                {
                    messageReceivedEventFailuresCounter.Increment();

                    logger.Error("A plugin encountered an error whilst handling the MessageReceived event. (See logs for exception)", e);
                }
                finally
                {
                    // Now we've executed everything, dispose the message reference and release the backing array!
                    messageReference.Dispose();
                    args.Dispose();
                }

                double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;

                messageReceivedEventTimeHistogram.Report(time);
            }

            //Inform plugins
            threadHelper.DispatchIfNeeded(DoMessageReceived);
        }
        /// <summary>
        ///     Called when a new client connects.
        /// </summary>
        /// <param name="connection">The new client.</param>
        internal void HandleNewConnection(NetworkServerConnection connection)
        {
            //Allocate ID and add to list
            ushort id;

            try
            {
                id = ReserveID();
            }
            catch (InvalidOperationException)
            {
                logger.Info($"New client could not be connected as there were no IDs available to allocate to them [{connection.RemoteEndPoints.Format()}].");

                connection.Disconnect();

                return;
            }


            Client client;

            try
            {
                client = Client.Create(
                    connection,
                    id,
                    this,
                    threadHelper,
                    clientLogger
#if PRO
                    , clientMetricsCollector
#endif
                    );
            }
            catch (Exception e)
            {
                logger.Error("An exception ocurred while connecting a client. The client has been dropped.", e);

                connection.Disconnect();

                DeallocateID(id, out int _);

                return;
            }

            AllocateIDToClient(id, client, out int noClients);

            // TODO if a client sends immediately after connecting then the message will be missed as the Connected event has not yet fired

            connection.Client = client;

            logger.Info($"New client [{client.ID}] connected [{client.RemoteEndPoints.Format()}].");
#if PRO
            clientsConnectedGauge.Report(noClients);
#endif

            //Inform plugins of the new connection
            EventHandler <ClientConnectedEventArgs> handler = ClientConnected;
            if (handler != null)
            {
                threadHelper.DispatchIfNeeded(
                    delegate()
                {
#if PRO
                    long startTimestamp = Stopwatch.GetTimestamp();
#endif
                    try
                    {
                        handler.Invoke(this, new ClientConnectedEventArgs(client));
                    }
                    catch (Exception e)
                    {
                        logger.Error("A plugin encountered an error whilst handling the ClientConnected event. The client will be disconnected. (See logs for exception)", e);

                        client.DropConnection();

#if PRO
                        clientConnectedEventFailuresCounter.Increment();
#endif
                        return;
                    }

#if PRO
                    double time = (double)(Stopwatch.GetTimestamp() - startTimestamp) / Stopwatch.Frequency;
                    clientConnectedEventTimeHistogram.Report(time);
#endif
                },
                    (_) => client.StartListening()
                    );
            }
        }