Exemple #1
0
        private void TcpConnectionEstablished(TcpPackageConnection connection)
        {
            if (_state != ConnectionState.Connecting || _connection != connection || connection.IsClosed)
            {
                LogDebug("IGNORED (_state {0}, _conn.Id {1:B}, conn.Id {2:B}, conn.closed {3}): TCP connection to [{4}, L{5}] established.",
                         _state, _connection == null ? Guid.Empty : _connection.ConnectionId, connection.ConnectionId,
                         connection.IsClosed, connection.RemoteEndPoint, connection.LocalEndPoint);
                return;
            }

            LogDebug("TCP connection to [{0}, L{1}, {2:B}] established.", connection.RemoteEndPoint, connection.LocalEndPoint, connection.ConnectionId);
            _heartbeatInfo = new HeartbeatInfo(_packageNumber, true, _stopwatch.Elapsed);

            if (_settings.DefaultUserCredentials != null)
            {
                _connectingPhase = ConnectingPhase.Authentication;

                _authInfo = new AuthInfo(Guid.NewGuid(), _stopwatch.Elapsed);
                _connection.EnqueueSend(new TcpPackage(TcpCommand.Authenticate,
                                                       TcpFlags.Authenticated,
                                                       _authInfo.CorrelationId,
                                                       _settings.DefaultUserCredentials.Username,
                                                       _settings.DefaultUserCredentials.Password,
                                                       null));
            }
            else
            {
                GoToConnectedState();
            }
        }
Exemple #2
0
 public HeartbeatConfigFromServer Info(HeartbeatInfo info)
 {
     tracker.OnInfo(info);
     return(new HeartbeatConfigFromServer
     {
         SendAtIntervals = true,
         SendChanges = true,
     });
 }
Exemple #3
0
        private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken)
        {
            const int     maxRetryCount = 2;
            HeartbeatInfo heartbeatInfo = null;

            for (var attempt = 1; attempt <= maxRetryCount; attempt++)
            {
                try
                {
                    if (_heartbeatConnection == null)
                    {
                        _heartbeatConnection = _heartbeatConnectionFactory.CreateConnection(_serverId, _endPoint);
                        await _heartbeatConnection.OpenAsync(TimeSpan.FromMinutes(1), cancellationToken);
                    }

                    heartbeatInfo = await GetHeartbeatInfoAsync(_heartbeatConnection, cancellationToken);

                    break;
                }
                catch
                {
                    _heartbeatConnection.Dispose();
                    _heartbeatConnection = null;

                    if (attempt == maxRetryCount)
                    {
                        _connectionPool.Clear();
                    }
                }
            }

            ServerDescription newDescription;

            if (heartbeatInfo != null)
            {
                var averageRoundTripTime        = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime);
                var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds));
                var isMasterResult  = heartbeatInfo.IsMasterResult;
                var buildInfoResult = heartbeatInfo.BuildInfoResult;

                newDescription = _baseDescription.WithHeartbeatInfo(
                    averageRoundTripTimeRounded,
                    isMasterResult.GetReplicaSetConfig(),
                    isMasterResult.Tags,
                    isMasterResult.ServerType,
                    buildInfoResult.ServerVersion,
                    new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion));
            }
            else
            {
                newDescription = _baseDescription;
            }

            OnDescriptionChanged(newDescription);

            return(true);
        }
Exemple #4
0
 public void Heartbeat(HeartbeatInfo heartbeat)
 {
     try
     {
         Trace.TraceInformation(JsonConvert.SerializeObject(heartbeat));
     }
     catch
     {
     }
 }
Exemple #5
0
        /// <summary>
        /// Updates the database with the heartbeat info of this executor
        /// </summary>
        /// <param name="info"></param>
        public void HeartbeatUpdate(HeartbeatInfo info)
        {
            // update ping time and other heartbeatinfo
            ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);

            executorStorage.PingTime       = DateTime.Now;
            executorStorage.Connected      = true;
            executorStorage.CpuUsage       = info.PercentUsedCpuPower;
            executorStorage.AvailableCpu   = info.PercentAvailCpuPower;
            executorStorage.TotalCpuUsage += info.Interval * (float)info.PercentUsedCpuPower / 100;

            ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
        }
        public object OnMessageReceived(object message, Server source, LocalShardHeartbeatReporting heartbeatReport, Membership existingMembership)
        {
            Election election = message as Election;

            if (election != null)
            {
                HeartbeatInfo localHeartbeat = heartbeatReport.GetHeartbeatInfo(_context.LocalAddress);
                if (localHeartbeat != null)
                {
                    return(_manager.CastVote(election, existingMembership, localHeartbeat));
                }
            }
            return(null);
        }
        public HeartbeatInfo GetPlatformHeartBeatInfo()
        {
            HeartbeatInfo hbi = new HeartbeatInfo();

            hbi.HomeId = Settings.HomeId;
            hbi.OrgId = Settings.OrgId;
            hbi.StudyId = Settings.StudyId;
            hbi.HubTimestamp = DateTime.UtcNow.ToString();
            hbi.PhysicalMemoryBytes = this.perfCountWorkingSet.NextValue();
            hbi.TotalCpuPercentage = this.perfCountPercentProcTime.NextValue();
            hbi.ModuleMonitorInfoList = this.platform.GetModuleMonitorInfoList();
            hbi.HeartbeatIntervalMins = this.heartbeatIntervalMins;
            hbi.SequenceNumber = this.sequenceNumber++;
            hbi.HardwareId = HomeOS.Hub.Common.Utils.HardwareId;

            return hbi;
        }
        public HeartbeatInfo GetPlatformHeartBeatInfo()
        {
            HeartbeatInfo hbi = new HeartbeatInfo();

            hbi.HomeId                = Settings.HomeId;
            hbi.OrgId                 = Settings.OrgId;
            hbi.StudyId               = Settings.StudyId;
            hbi.HubTimestamp          = DateTime.UtcNow.ToString();
            hbi.PhysicalMemoryBytes   = this.perfCountWorkingSet.NextValue();
            hbi.TotalCpuPercentage    = this.perfCountPercentProcTime.NextValue();
            hbi.ModuleMonitorInfoList = this.platform.GetModuleMonitorInfoList();
            hbi.ScoutInfoList         = this.platform.GetScoutInfoList();
            hbi.HeartbeatIntervalMins = this.heartbeatIntervalMins;
            hbi.SequenceNumber        = this.sequenceNumber++;
            hbi.HardwareId            = HomeOS.Hub.Common.Utils.HardwareId;
            hbi.PlatformVersion       = this.platform.GetPlatformVersion();

            return(hbi);
        }
 private void SendHeartbeat(Object stateInfo)
 {
     try
     {
         HeartbeatInfo heartbeatInfo = GetPlatformHeartBeatInfo();
         if (null != heartbeatInfo)
         {
             string jsonString = heartbeatInfo.SerializeToJsonStream();
             logger.Log("Sending heartbeat: {0}", jsonString);
             var webClient = new NoKeepAlivesWebClient();
             webClient.UploadStringCompleted  += webClient_UploadHeartbeatInfoCompleted;
             webClient.Headers["Content-type"] = "application/json";
             webClient.Encoding = Encoding.UTF8;
             webClient.UseDefaultCredentials = true;
             webClient.UploadStringAsync(new Uri(this.uri.OriginalString + "/SetHeartbeatInfo"), "POST", jsonString, jsonString);
         }
     }
     catch (Exception e)
     {
         logger.Log("Failed to send heartbeat, exception={0}", e.Message);
     }
 }
Exemple #10
0
        private void ManageHeartbeats()
        {
            if (_connection == null)
            {
                throw new Exception();
            }

            var timeout = _heartbeatInfo.IsIntervalStage ? _settings.HeartbeatInterval : _settings.HeartbeatTimeout;

            if (_stopwatch.Elapsed - _heartbeatInfo.TimeStamp < timeout)
            {
                return;
            }

            var packageNumber = _packageNumber;

            if (_heartbeatInfo.LastPackageNumber != packageNumber)
            {
                _heartbeatInfo = new HeartbeatInfo(packageNumber, true, _stopwatch.Elapsed);
                return;
            }

            if (_heartbeatInfo.IsIntervalStage)
            {
                // TcpMessage.Heartbeat analog
                _connection.EnqueueSend(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null));
                _heartbeatInfo = new HeartbeatInfo(_heartbeatInfo.LastPackageNumber, false, _stopwatch.Elapsed);
            }
            else
            {
                // TcpMessage.HeartbeatTimeout analog
                var msg = string.Format(
                    "EventStoreConnection '{0}': closing TCP connection [{1}, {2}, {3}] due to HEARTBEAT TIMEOUT at pkgNum {4}.",
                    _esConnection.ConnectionName, _connection.RemoteEndPoint, _connection.LocalEndPoint,
                    _connection.ConnectionId, packageNumber);
                _settings.Log.Info(msg);
                CloseTcpConnection(msg);
            }
        }
        public void SetHeartbeatInfo(HeartbeatInfo hbi)
        {
            if (null == hbi.OrgId || hbi.OrgId.Length == 0)
            {
                hbi.OrgId = "Unknown";
            }

            HeartbeatTable HeartbeatTable = Helper.GetHeartbeatTable();

            if (null == hbi)
            {
                throw new ArgumentNullException();
            }

            try
            {
                HeartbeatTable.Write(hbi.OrgId, hbi.HomeId, hbi.SerializeToJsonStream());
            }
            catch(Exception e)
            {
                Helper.Trace().WriteLine("SimpleStorage.Write Failed with Exception:{0}", e.Message);
            }
        }
        private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken)
        {
            const int     maxRetryCount      = 2;
            HeartbeatInfo heartbeatInfo      = null;
            Exception     heartbeatException = null;

            for (var attempt = 1; attempt <= maxRetryCount; attempt++)
            {
                try
                {
                    if (_heartbeatConnection == null)
                    {
                        _heartbeatConnection = _heartbeatConnectionFactory.CreateConnection(_serverId, _endPoint);
                        // if we are cancelling, it's because the server has
                        // been shut down and we really don't need to wait.
                        await _heartbeatConnection.OpenAsync(cancellationToken).ConfigureAwait(false);
                    }

                    heartbeatInfo = await GetHeartbeatInfoAsync(_heartbeatConnection, cancellationToken).ConfigureAwait(false);

                    heartbeatException = null;
                    break;
                }
                catch (Exception ex)
                {
                    heartbeatException = ex;
                    _heartbeatConnection.Dispose();
                    _heartbeatConnection = null;

                    if (attempt == maxRetryCount)
                    {
                        _connectionPool.Clear();
                    }
                }
            }

            ServerDescription newDescription;

            if (heartbeatInfo != null)
            {
                var averageRoundTripTime        = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime);
                var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds));
                var isMasterResult  = heartbeatInfo.IsMasterResult;
                var buildInfoResult = heartbeatInfo.BuildInfoResult;

                newDescription = _baseDescription.With(
                    averageRoundTripTime: averageRoundTripTimeRounded,
                    canonicalEndPoint: isMasterResult.Me,
                    electionId: isMasterResult.ElectionId,
                    maxBatchCount: isMasterResult.MaxBatchCount,
                    maxDocumentSize: isMasterResult.MaxDocumentSize,
                    maxMessageSize: isMasterResult.MaxMessageSize,
                    replicaSetConfig: isMasterResult.GetReplicaSetConfig(),
                    state: ServerState.Connected,
                    tags: isMasterResult.Tags,
                    type: isMasterResult.ServerType,
                    version: buildInfoResult.ServerVersion,
                    wireVersionRange: new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion));
            }
            else
            {
                newDescription = _baseDescription;
            }

            if (heartbeatException != null)
            {
                newDescription = newDescription.With(heartbeatException: heartbeatException);
            }

            OnDescriptionChanged(newDescription);

            return(true);
        }
Exemple #13
0
        private void Heartbeat()
        {
            int pingFailCount = 0;

            logger.Info("HeartBeat Thread Started.");

            //heart-beat thread handles its own errors.
            try
            {
                HeartbeatInfo info = new HeartbeatInfo();
                info.Interval = _HeartbeatInterval;

                // init for cpu usage
                TimeSpan oldTime      = Process.GetCurrentProcess().TotalProcessorTime;
                DateTime timeMeasured = DateTime.Now;
                TimeSpan newTime      = new TimeSpan(0);

                //TODO: be language neutral here. how??
                // init for cpu avail
                PerformanceCounter cpuCounter = new PerformanceCounter();
                cpuCounter.ReadOnly     = true;
                cpuCounter.CategoryName = "Processor";
                cpuCounter.CounterName  = "% Processor Time";
                cpuCounter.InstanceName = "_Total";

                while (!_stop)
                {
                    // calculate usage
                    newTime = Process.GetCurrentProcess().TotalProcessorTime;
                    TimeSpan absUsage = newTime - oldTime;
                    float    flUsage  = ((float)absUsage.Ticks / (DateTime.Now - timeMeasured).Ticks) * 100;
                    info.PercentUsedCpuPower = (int)flUsage > 100 ? 100 : (int)flUsage;
                    info.PercentUsedCpuPower = (int)flUsage < 0 ? 0 : (int)flUsage;
                    timeMeasured             = DateTime.Now;
                    oldTime = newTime;

                    try
                    {
                        // calculate avail
                        info.PercentAvailCpuPower = 100 - (int)cpuCounter.NextValue() + info.PercentUsedCpuPower;
                        info.PercentAvailCpuPower = info.PercentAvailCpuPower > 100 ? 100 : info.PercentAvailCpuPower;
                        info.PercentAvailCpuPower = info.PercentAvailCpuPower < 0 ? 0 : info.PercentAvailCpuPower;
                    }
                    catch (Exception e)
                    {
                        logger.Debug("HeartBeat thread error: " + e.Message + Environment.NewLine + " Heartbeat continuing after error...");
                    }

                    //have a seperate try...block since we try 3 times before giving up
                    try
                    {
                        //send a heartbeat to the manager.
                        _executor.Manager.Executor_Heartbeat(_executor.Credentials, _executor.Id, info);
                        pingFailCount = 0;
                    }
                    catch (Exception e)
                    {
                        if (e is SocketException || e is System.Runtime.Remoting.RemotingException)
                        {
                            pingFailCount++;
                            //we disconnect the executor if the manager cant be pinged three times
                            if (pingFailCount >= 3)
                            {
                                logger.Error("Failed to contact manager " + pingFailCount + " times...", e);

                                //the disconnect here should be started off on a seperate thread because:
                                //disconnect itself waits for HeartBeatThread to stop. If the method call
                                //to disconnect from HeartBeat wont return immediately, then there is a deadlock
                                //with disconnect waiting for the HeartBeatThread to stop and the HeartBeatThread waiting
                                //for the call to disconnect to return.

                                //raise the event to indicate that the Executor has got disconnected.
                                _executor.OnGotDisconnected();
                            }
                        }
                        else if (e is InvalidExecutorException)
                        {
                            //raise a disconnected event.
                            //so that we will reconnect if needed.
                            logger.Error("Invalid Executor exception : ", e);
                            _executor.OnGotDisconnected();
                        }
                        else
                        {
                            logger.Debug("Error during heartbeat. Continuing after error...", e);
                        }
                    }

                    Thread.Sleep(_HeartbeatInterval * 1000);
                }
            }
            catch (ThreadAbortException)
            {
                Thread.ResetAbort();
                logger.Info("HeartBeat thread aborted.");
            }
            catch (Exception e)
            {
                logger.Error("HeartBeat Exception. Heartbeat thread stopping...", e);
            }

            logger.Info("HeartBeat Thread Exited.");
        }
Exemple #14
0
        private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken)
        {
            const int     maxRetryCount      = 2;
            HeartbeatInfo heartbeatInfo      = null;
            Exception     heartbeatException = null;

            for (var attempt = 1; attempt <= maxRetryCount; attempt++)
            {
                var connection = _connection;
                try
                {
                    if (connection == null)
                    {
                        connection = _connectionFactory.CreateConnection(_serverId, _endPoint);
                        // if we are cancelling, it's because the server has
                        // been shut down and we really don't need to wait.
                        await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
                    }

                    heartbeatInfo = await GetHeartbeatInfoAsync(connection, cancellationToken).ConfigureAwait(false);

                    heartbeatException = null;

                    _connection = connection;
                    break;
                }
                catch (Exception ex)
                {
                    heartbeatException = ex;
                    _connection        = null;
                    if (connection != null)
                    {
                        connection.Dispose();
                    }
                }
            }

            ServerDescription newDescription;

            if (heartbeatInfo != null)
            {
                var averageRoundTripTime        = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime);
                var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds));
                var isMasterResult  = heartbeatInfo.IsMasterResult;
                var buildInfoResult = heartbeatInfo.BuildInfoResult;

                newDescription = _baseDescription.With(
                    averageRoundTripTime: averageRoundTripTimeRounded,
                    canonicalEndPoint: isMasterResult.Me,
                    electionId: isMasterResult.ElectionId,
                    lastWriteTimestamp: isMasterResult.LastWriteTimestamp,
                    logicalSessionTimeout: isMasterResult.LogicalSessionTimeout,
                    maxBatchCount: isMasterResult.MaxBatchCount,
                    maxDocumentSize: isMasterResult.MaxDocumentSize,
                    maxMessageSize: isMasterResult.MaxMessageSize,
                    replicaSetConfig: isMasterResult.GetReplicaSetConfig(),
                    state: ServerState.Connected,
                    tags: isMasterResult.Tags,
                    topologyVersion: isMasterResult.TopologyVersion,
                    type: isMasterResult.ServerType,
                    version: buildInfoResult.ServerVersion,
                    wireVersionRange: new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion));
            }
            else
            {
                newDescription = _baseDescription.With(lastUpdateTimestamp: DateTime.UtcNow);
            }

            if (heartbeatException != null)
            {
                var topologyVersion = default(Optional <TopologyVersion>);
                if (heartbeatException is MongoCommandException heartbeatCommandException)
                {
                    topologyVersion = TopologyVersion.FromMongoCommandException(heartbeatCommandException);
                }
                newDescription = newDescription.With(heartbeatException: heartbeatException, topologyVersion: topologyVersion);
            }

            newDescription = newDescription.With(reasonChanged: "Heartbeat", lastHeartbeatTimestamp: DateTime.UtcNow);

            SetDescription(newDescription);

            return(true);
        }
Exemple #15
0
        protected VariableParam readVariable()
        {
            int type = _body.GetUShort();
            int len  = _body.GetUShort();
            int blen = len - 4;
            //byte[] data;
            Unknown var;

            switch (type)
            {
            case 1:
                var = new HeartbeatInfo(1, "HeartbeatInfo");
                break;

            //      2-4	Unassigned
            case 5:
                var = new IPv4Address(5, "IPv4Address");
                break;

            case 6:
                var = new IPv6Address(6, "IPv6Address");
                break;

            case 7:
                var = new StateCookie(7, "StateCookie");
                break;

            case 8:
                var = new UnrecognizedParameters(8, "UnrecognizedParameters");
                break;

            case 9:
                var = new CookiePreservative(9, "CookiePreservative");
                break;

            //      10	Unassigned
            case 11:
                var = new HostNameAddress(11, "HostNameAddress");
                break;

            case 12:
                var = new SupportedAddressTypes(12, "SupportedAddressTypes");
                break;

            case 13:
                var = new OutgoingSSNResetRequestParameter(13, "OutgoingSSNResetRequestParameter");
                break;

            case 14:
                var = new IncomingSSNResetRequestParameter(14, "IncomingSSNResetRequestParameter");
                break;

            case 15:
                var = new SSNTSNResetRequestParameter(15, "SSNTSNResetRequestParameter");
                break;

            case 16:
                var = new ReconfigurationResponseParameter(16, "ReconfigurationResponseParameter");
                break;

            case 17:
                var = new AddOutgoingStreamsRequestParameter(17, "AddOutgoingStreamsRequestParameter");
                break;

            case 18:
                var = new AddIncomingStreamsRequestParameter(18, "AddIncomingStreamsRequestParameter");
                break;

            //      19-32767	Unassigned
            case 32768:
                var = new Unknown(32768, "ReservedforECNCapable");
                break;

            case 32770:
                var = new RandomParam(32770, "Random");
                break;

            case 32771:
                var = new ChunkListParam(32771, "ChunkList");
                break;

            case 32772:
                var = new RequestedHMACAlgorithmParameter(32772, "RequestedHMACAlgorithmParameter");
                break;

            case 32773:
                var = new Unknown(32773, "Padding");
                break;

            case 32776:
                var = new SupportedExtensions(32776, "SupportedExtensions");
                break;

            //      32777-49151	Unassigned
            case 49152:
                var = new ForwardTSNsupported(49152, "ForwardTSNsupported");
                break;

            case 49153:
                var = new Unknown(49153, "AddIPAddress");
                break;

            case 49154:
                var = new Unknown(49154, "DeleteIPAddress");
                break;

            case 49155:
                var = new Unknown(49155, "ErrorCauseIndication");
                break;

            case 49156:
                var = new Unknown(49156, "SetPrimaryAddress");
                break;

            case 49157:
                var = new Unknown(49157, "SuccessIndication");
                break;

            case 49158:
                var = new Unknown(49158, "AdaptationLayerIndication");
                break;

            default:
                var = new Unknown(-1, "Unknown");
                break;
            }
            try
            {
                var.readBody(_body, blen);
                //logger.LogDebug("variable type " + var.getType() + " name " + var.getName());
            }
            catch (SctpPacketFormatException ex)
            {
                logger.LogError(ex.ToString());
            }
            if (_body.hasRemaining())
            {
                int mod = blen % 4;
                if (mod != 0)
                {
                    for (int pad = mod; pad < 4; pad++)
                    {
                        _body.GetByte();
                    }
                }
            }
            return(var);
        }
Exemple #16
0
        internal ElectionVote CastVote(Election election, Membership existingMembership, HeartbeatInfo info)
        {
            if (election == null || election.ElectionId == null)
            {
                if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
                {
                    LoggerManager.Instance.ShardLogger.Warn("ElectionManager.CastVote()", "Invalid election.");
                }
                return(null);
            }
            ElectionVote vote = new ElectionVote();

            vote.RequestingNode      = election.ElectionId.RequestingNode;
            vote.Sourcenode          = new ServerNode();
            vote.Sourcenode.Name     = _context.LocalAddress.IpAddress.ToString();
            vote.Sourcenode.Priority = (int)LocalServerPriority;
            if (existingMembership != null && (election.ElectionType == ElectionType.GeneralElections && existingMembership.Primary == null) || election.ElectionType == ElectionType.TakeoverElections)
            {
                if (election.RequestingServerInfo.LastOperationId > LocalServerInfo().LastOperationId)
                {
                    vote.NodeVote = ElectionVote.Vote.yes;
                }
                else if (election.RequestingServerInfo.LastOperationId == LocalServerInfo().LastOperationId)
                {
                    if (election.ElectionId.RequestingNode.Priority <= (int)LocalServerPriority)
                    {
                        vote.NodeVote = ElectionVote.Vote.yes;
                    }
                    else
                    {
                        ShardConfiguration sConfig = null;
                        if (_clusterConfigMgr != null)
                        {
                            sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
                        }
                        int configuredNodes = 0;
                        if (sConfig != null || sConfig.Servers != null)
                        {
                            configuredNodes = sConfig.Servers.Nodes.Count;
                        }
                        if (info != null && info.CSStatus == ConnectivityStatus.Connected && (configuredNodes != null && existingMembership.Servers.Count >= configuredNodes / 2))
                        {
                            vote.NodeVote = ElectionVote.Vote.no;
                        }
                        else
                        {
                            vote.NodeVote = ElectionVote.Vote.yes;
                        }
                    }
                }
                else
                {
                    vote.NodeVote = ElectionVote.Vote.no;
                }
            }
            else
            {
                vote.NodeVote = ElectionVote.Vote.no;
            }

            return(vote);
        }
        private void TcpConnectionEstablished(TcpPackageConnection connection)
        {
            if (_state != ConnectionState.Connecting || _connection != connection || connection.IsClosed)
            {
                LogDebug("IGNORED (_state {0}, _conn.Id {1:B}, conn.Id {2:B}, conn.closed {3}): TCP connection to [{4}, L{5}] established.", 
                         _state, _connection == null ? Guid.Empty : _connection.ConnectionId, connection.ConnectionId, 
                         connection.IsClosed, connection.RemoteEndPoint, connection.LocalEndPoint);
                return;
            }

            LogDebug("TCP connection to [{0}, L{1}, {2:B}] established.", connection.RemoteEndPoint, connection.LocalEndPoint, connection.ConnectionId);
            _heartbeatInfo = new HeartbeatInfo(_packageNumber, true, _stopwatch.Elapsed);

            if (_settings.DefaultUserCredentials != null)
            {
                _connectingPhase = ConnectingPhase.Authentication;

                _authInfo = new AuthInfo(Guid.NewGuid(), _stopwatch.Elapsed);
                _connection.EnqueueSend(new TcpPackage(TcpCommand.Authenticate,
                                                       TcpFlags.Authenticated,
                                                       _authInfo.CorrelationId,
                                                       _settings.DefaultUserCredentials.Username,
                                                       _settings.DefaultUserCredentials.Password, 
                                                       null));
            }
            else
            {
                GoToConnectedState();
            }
        }
        /// <summary>
        /// Speculative Phase:
        /// 1. We search for the node with the latest op-log entry. This detail bears the highest value.
        /// 2. Next, if multiple nodes have the same op-log entry, we move onto the next step.
        /// 3. All those nodes which have the same op-log entry AND are connected to the CS are considered.
        /// 4. Highest priority from amongst these active nodes are taken into consideration.
        /// 5. If this node fulfills all of the above, it successfully passes the speculative phase.
        /// </summary>
        /// <param name="heartbeatReport"></param>
        /// <returns></returns>

        private bool ShouldIInitiateElection(LocalShardHeartbeatReporting heartbeatReport, Activity activity)
        {
            IList <Address> activeNodes = null;

            if (heartbeatReport != null)
            {
                activeNodes = heartbeatReport.GetReportTable.Keys.ToList();
                OperationId    maxOplog = null;
                IList <string> matchingOplogServerIPs = new List <string>();

                HeartbeatInfo localHeartbeat = heartbeatReport.GetHeartbeatInfo(_context.LocalAddress);
                if (localHeartbeat == null)
                {
                    if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
                    {
                        LoggerManager.Instance.ShardLogger.Warn(
                            "ElectionBasedMembershipStrategy.ShouldIInititateElections()",
                            "local node heartbeat is null");
                    }
                    return(false);
                }

                OperationId lastRepId = null;


                if (activity.Equals(Activity.TakeoverElectionsTriggered))
                {
                    HeartbeatInfo info = null;
                    if (_shard != null && _shard.Primary != null)
                    {
                        info = heartbeatReport.GetHeartbeatInfo(_shard.Primary.Address);
                    }
                }

                else
                {
                    ShardConfiguration sConfig       = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
                    int         configuredNodesCount = 0;
                    OperationId OpIdAtCS             = null;
                    if (sConfig != null && sConfig.Servers != null && sConfig.Servers.Nodes != null)
                    {
                        configuredNodesCount = sConfig.Servers.Nodes.Count;
                    }
                    if (configuredNodesCount > 0 && activeNodes != null && activeNodes.Count < configuredNodesCount)
                    {
                        ShardInfo   sInfo = null;
                        ClusterInfo cInfo = _context.ConfigurationSession.GetDatabaseClusterInfo(_context.ClusterName);
                        if (cInfo != null)
                        {
                            sInfo = cInfo.GetShardInfo(_context.LocalShardName);
                        }
                        if (sInfo != null)
                        {
                            OpIdAtCS = sInfo.LastOperationId;
                        }
                        if (OpIdAtCS > lastRepId)
                        {
                            if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
                            {
                                LoggerManager.Instance.ShardLogger.Info(
                                    "electBasedMemSt.ShouldIInitElections()",
                                    "CS has an operation newer than my operation. Hence, waiting.");
                            }

                            if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
                            {
                                if (OpIdAtCS != null)
                                {
                                    LoggerManager.Instance.ShardLogger.Info(
                                        "electBasedMemSt.ShouldIInitElections()",
                                        "Operation ID on the CS:- " + OpIdAtCS.ElectionId + ":"
                                        + OpIdAtCS.ElectionBasedSequenceId);
                                }
                                else
                                {
                                    LoggerManager.Instance.ShardLogger.Info(
                                        "electBasedMemSt.ShouldIInitElections()", "The operation ID at the CS is set to null.");
                                }
                                if (lastRepId != null)
                                {
                                    LoggerManager.Instance.ShardLogger.Info(
                                        "electBasedMemSt.ShouldIInitElections()", "Local node Operation ID:- " + lastRepId.ElectionId +
                                        ":" + lastRepId.ElectionBasedSequenceId);
                                }
                                else
                                {
                                    LoggerManager.Instance.ShardLogger.Info(
                                        "electBasedMemSt.ShouldIInitElections()", "The local node operation ID is set to null.");
                                }
                            }
                            //We maintain the last replicated operation log entry with the CS.
                            //If a node in a shard with older data (usually the previous secondary) is up before the node with the
                            //latest data(usually the previous primary), it waits for a configurable amount of time (2 minutes for
                            //now) before proceeding with the election procedure if it is still unable to detect a primary node.
                            //This way we give the node with the latest data a chance to become primary and therefore avoid data loss.
                            lock (_mutexOnWait)
                            {
                                Monitor.Wait(_mutexOnWait, _waitTimeout);
                            }
                        }
                    }

                    for (int i = 0; i < activeNodes.Count; i++)
                    {
                        HeartbeatInfo info           = heartbeatReport.GetHeartbeatInfo(activeNodes[i]);
                        OperationId   currIndexOplog = info.LastOplogOperationId;
                        if (currIndexOplog > maxOplog)
                        {
                            maxOplog = currIndexOplog;
                        }

                        if (((localHeartbeat.LastOplogOperationId == null && info.LastOplogOperationId == null) || localHeartbeat.LastOplogOperationId != null && localHeartbeat.LastOplogOperationId.Equals(info.LastOplogOperationId)) &&
                            info.CSStatus == ConnectivityStatus.Connected)
                        {
                            matchingOplogServerIPs.Add(activeNodes[i].IpAddress.ToString());
                        }
                    }
                }

                if (localHeartbeat.LastOplogOperationId != null && maxOplog != null && maxOplog > localHeartbeat.LastOplogOperationId && (lastRepId == null || maxOplog > lastRepId))
                {
                    if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
                    {
                        LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "Local operation log is behind max op log wrt " + _context.LocalShardName + " shard.");
                    }
                    if (maxOplog != null && LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
                    {
                        LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "maxOplog: " + maxOplog.ElectionId + ":" + maxOplog.ElectionBasedSequenceId);
                    }
                    if (localHeartbeat.LastOplogOperationId != null && LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
                    {
                        LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "local opLog (from the heartbeat): " + localHeartbeat.LastOplogOperationId.ElectionId + ":" + localHeartbeat.LastOplogOperationId.ElectionBasedSequenceId);
                    }
                    if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled && lastRepId != null)
                    {
                        LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "LastOpFromOpLog (from the replication module): " + lastRepId.ElectionId + ":" + lastRepId.ElectionBasedSequenceId);
                    }

                    return(false);
                }
                else if (maxOplog == localHeartbeat.LastOplogOperationId || (lastRepId != null && lastRepId.Equals(maxOplog)))
                {
                    //if: there are multiple nodes that have the same oplog entry,
                    //decision will be made on the basis of the priorities.
                    //else: the node with the highest oplog entry will be considered eligible.
                    if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
                    {
                        LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "Local operation log is equal to the max op log wrt " + _context.LocalShardName + " shard.");
                    }
                    if (maxOplog != null && (localHeartbeat != null && localHeartbeat.LastOplogOperationId != null))
                    {
                        if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
                        {
                            LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "maxOplog: " + maxOplog.ElectionId + ":" + maxOplog.ElectionBasedSequenceId);
                        }
                        if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
                        {
                            LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "local opLog (from the heartbeat): " + localHeartbeat.LastOplogOperationId.ElectionId + ":" + localHeartbeat.LastOplogOperationId.ElectionBasedSequenceId);
                        }
                        if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled && lastRepId != null)
                        {
                            LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "LastOpFromOpLog (from the replication module): " + lastRepId.ElectionId + ":" + lastRepId.ElectionBasedSequenceId);
                        }
                    }
                    if (matchingOplogServerIPs.Count > 0)
                    {
                        int highestRunningNodePriority = _manager.GetHighestNodePriority(activeNodes,
                                                                                         matchingOplogServerIPs);

                        if (highestRunningNodePriority.Equals(_manager.LocalServerPriority))
                        {
                            if (LoggerManager.Instance.ShardLogger != null &&
                                LoggerManager.Instance.ShardLogger.IsDebugEnabled)
                            {
                                LoggerManager.Instance.ShardLogger.Debug(
                                    "ElectionBasedMembershipStrategy.ShouldIInitiateElection()",
                                    "Node : " + _context.LocalAddress.IpAddress.ToString() + " in shard: " +
                                    _context.LocalShardName + " is eligible having priority: " +
                                    highestRunningNodePriority + " .");
                            }
                            return(true);
                        }
                        else
                        {
                            return(false);
                        }
                    }
                }
            }
            return(false);
        }
        private void ManageHeartbeats()
        {
            if (_connection == null) throw new Exception();

            var timeout = _heartbeatInfo.IsIntervalStage ? _settings.HeartbeatInterval : _settings.HeartbeatTimeout;
            if (_stopwatch.Elapsed - _heartbeatInfo.TimeStamp < timeout) 
                return;

            var packageNumber = _packageNumber;
            if (_heartbeatInfo.LastPackageNumber != packageNumber)
            {
                _heartbeatInfo = new HeartbeatInfo(packageNumber, true, _stopwatch.Elapsed);
                return;
            }

            if (_heartbeatInfo.IsIntervalStage)
            {
                // TcpMessage.Heartbeat analog
                _connection.EnqueueSend(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null));
                _heartbeatInfo = new HeartbeatInfo(_heartbeatInfo.LastPackageNumber, false, _stopwatch.Elapsed);
            }
            else 
            {
                // TcpMessage.HeartbeatTimeout analog
                var msg = string.Format("EventStoreConnection '{0}': closing TCP connection [{1}, L{2}, {3:B}] due to HEARTBEAT TIMEOUT at pkgNum {4}.",
                                        _esConnection.ConnectionName, _connection.RemoteEndPoint, _connection.LocalEndPoint,
                                        _connection.ConnectionId, packageNumber);
                _settings.Log.Info(msg);
                CloseTcpConnection(msg);
            }
        }
 public Task Heartbeat(HeartbeatInfo heartbeat)
 {
     return(Task.Factory.StartNew(() => HeartbeatService.Heartbeat(heartbeat)));
 }