private void TcpConnectionEstablished(TcpPackageConnection connection) { if (_state != ConnectionState.Connecting || _connection != connection || connection.IsClosed) { LogDebug("IGNORED (_state {0}, _conn.Id {1:B}, conn.Id {2:B}, conn.closed {3}): TCP connection to [{4}, L{5}] established.", _state, _connection == null ? Guid.Empty : _connection.ConnectionId, connection.ConnectionId, connection.IsClosed, connection.RemoteEndPoint, connection.LocalEndPoint); return; } LogDebug("TCP connection to [{0}, L{1}, {2:B}] established.", connection.RemoteEndPoint, connection.LocalEndPoint, connection.ConnectionId); _heartbeatInfo = new HeartbeatInfo(_packageNumber, true, _stopwatch.Elapsed); if (_settings.DefaultUserCredentials != null) { _connectingPhase = ConnectingPhase.Authentication; _authInfo = new AuthInfo(Guid.NewGuid(), _stopwatch.Elapsed); _connection.EnqueueSend(new TcpPackage(TcpCommand.Authenticate, TcpFlags.Authenticated, _authInfo.CorrelationId, _settings.DefaultUserCredentials.Username, _settings.DefaultUserCredentials.Password, null)); } else { GoToConnectedState(); } }
public HeartbeatConfigFromServer Info(HeartbeatInfo info) { tracker.OnInfo(info); return(new HeartbeatConfigFromServer { SendAtIntervals = true, SendChanges = true, }); }
private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken) { const int maxRetryCount = 2; HeartbeatInfo heartbeatInfo = null; for (var attempt = 1; attempt <= maxRetryCount; attempt++) { try { if (_heartbeatConnection == null) { _heartbeatConnection = _heartbeatConnectionFactory.CreateConnection(_serverId, _endPoint); await _heartbeatConnection.OpenAsync(TimeSpan.FromMinutes(1), cancellationToken); } heartbeatInfo = await GetHeartbeatInfoAsync(_heartbeatConnection, cancellationToken); break; } catch { _heartbeatConnection.Dispose(); _heartbeatConnection = null; if (attempt == maxRetryCount) { _connectionPool.Clear(); } } } ServerDescription newDescription; if (heartbeatInfo != null) { var averageRoundTripTime = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime); var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds)); var isMasterResult = heartbeatInfo.IsMasterResult; var buildInfoResult = heartbeatInfo.BuildInfoResult; newDescription = _baseDescription.WithHeartbeatInfo( averageRoundTripTimeRounded, isMasterResult.GetReplicaSetConfig(), isMasterResult.Tags, isMasterResult.ServerType, buildInfoResult.ServerVersion, new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion)); } else { newDescription = _baseDescription; } OnDescriptionChanged(newDescription); return(true); }
public void Heartbeat(HeartbeatInfo heartbeat) { try { Trace.TraceInformation(JsonConvert.SerializeObject(heartbeat)); } catch { } }
/// <summary> /// Updates the database with the heartbeat info of this executor /// </summary> /// <param name="info"></param> public void HeartbeatUpdate(HeartbeatInfo info) { // update ping time and other heartbeatinfo ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id); executorStorage.PingTime = DateTime.Now; executorStorage.Connected = true; executorStorage.CpuUsage = info.PercentUsedCpuPower; executorStorage.AvailableCpu = info.PercentAvailCpuPower; executorStorage.TotalCpuUsage += info.Interval * (float)info.PercentUsedCpuPower / 100; ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage); }
public object OnMessageReceived(object message, Server source, LocalShardHeartbeatReporting heartbeatReport, Membership existingMembership) { Election election = message as Election; if (election != null) { HeartbeatInfo localHeartbeat = heartbeatReport.GetHeartbeatInfo(_context.LocalAddress); if (localHeartbeat != null) { return(_manager.CastVote(election, existingMembership, localHeartbeat)); } } return(null); }
public HeartbeatInfo GetPlatformHeartBeatInfo() { HeartbeatInfo hbi = new HeartbeatInfo(); hbi.HomeId = Settings.HomeId; hbi.OrgId = Settings.OrgId; hbi.StudyId = Settings.StudyId; hbi.HubTimestamp = DateTime.UtcNow.ToString(); hbi.PhysicalMemoryBytes = this.perfCountWorkingSet.NextValue(); hbi.TotalCpuPercentage = this.perfCountPercentProcTime.NextValue(); hbi.ModuleMonitorInfoList = this.platform.GetModuleMonitorInfoList(); hbi.HeartbeatIntervalMins = this.heartbeatIntervalMins; hbi.SequenceNumber = this.sequenceNumber++; hbi.HardwareId = HomeOS.Hub.Common.Utils.HardwareId; return hbi; }
public HeartbeatInfo GetPlatformHeartBeatInfo() { HeartbeatInfo hbi = new HeartbeatInfo(); hbi.HomeId = Settings.HomeId; hbi.OrgId = Settings.OrgId; hbi.StudyId = Settings.StudyId; hbi.HubTimestamp = DateTime.UtcNow.ToString(); hbi.PhysicalMemoryBytes = this.perfCountWorkingSet.NextValue(); hbi.TotalCpuPercentage = this.perfCountPercentProcTime.NextValue(); hbi.ModuleMonitorInfoList = this.platform.GetModuleMonitorInfoList(); hbi.ScoutInfoList = this.platform.GetScoutInfoList(); hbi.HeartbeatIntervalMins = this.heartbeatIntervalMins; hbi.SequenceNumber = this.sequenceNumber++; hbi.HardwareId = HomeOS.Hub.Common.Utils.HardwareId; hbi.PlatformVersion = this.platform.GetPlatformVersion(); return(hbi); }
private void SendHeartbeat(Object stateInfo) { try { HeartbeatInfo heartbeatInfo = GetPlatformHeartBeatInfo(); if (null != heartbeatInfo) { string jsonString = heartbeatInfo.SerializeToJsonStream(); logger.Log("Sending heartbeat: {0}", jsonString); var webClient = new NoKeepAlivesWebClient(); webClient.UploadStringCompleted += webClient_UploadHeartbeatInfoCompleted; webClient.Headers["Content-type"] = "application/json"; webClient.Encoding = Encoding.UTF8; webClient.UseDefaultCredentials = true; webClient.UploadStringAsync(new Uri(this.uri.OriginalString + "/SetHeartbeatInfo"), "POST", jsonString, jsonString); } } catch (Exception e) { logger.Log("Failed to send heartbeat, exception={0}", e.Message); } }
private void ManageHeartbeats() { if (_connection == null) { throw new Exception(); } var timeout = _heartbeatInfo.IsIntervalStage ? _settings.HeartbeatInterval : _settings.HeartbeatTimeout; if (_stopwatch.Elapsed - _heartbeatInfo.TimeStamp < timeout) { return; } var packageNumber = _packageNumber; if (_heartbeatInfo.LastPackageNumber != packageNumber) { _heartbeatInfo = new HeartbeatInfo(packageNumber, true, _stopwatch.Elapsed); return; } if (_heartbeatInfo.IsIntervalStage) { // TcpMessage.Heartbeat analog _connection.EnqueueSend(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null)); _heartbeatInfo = new HeartbeatInfo(_heartbeatInfo.LastPackageNumber, false, _stopwatch.Elapsed); } else { // TcpMessage.HeartbeatTimeout analog var msg = string.Format( "EventStoreConnection '{0}': closing TCP connection [{1}, {2}, {3}] due to HEARTBEAT TIMEOUT at pkgNum {4}.", _esConnection.ConnectionName, _connection.RemoteEndPoint, _connection.LocalEndPoint, _connection.ConnectionId, packageNumber); _settings.Log.Info(msg); CloseTcpConnection(msg); } }
public void SetHeartbeatInfo(HeartbeatInfo hbi) { if (null == hbi.OrgId || hbi.OrgId.Length == 0) { hbi.OrgId = "Unknown"; } HeartbeatTable HeartbeatTable = Helper.GetHeartbeatTable(); if (null == hbi) { throw new ArgumentNullException(); } try { HeartbeatTable.Write(hbi.OrgId, hbi.HomeId, hbi.SerializeToJsonStream()); } catch(Exception e) { Helper.Trace().WriteLine("SimpleStorage.Write Failed with Exception:{0}", e.Message); } }
private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken) { const int maxRetryCount = 2; HeartbeatInfo heartbeatInfo = null; Exception heartbeatException = null; for (var attempt = 1; attempt <= maxRetryCount; attempt++) { try { if (_heartbeatConnection == null) { _heartbeatConnection = _heartbeatConnectionFactory.CreateConnection(_serverId, _endPoint); // if we are cancelling, it's because the server has // been shut down and we really don't need to wait. await _heartbeatConnection.OpenAsync(cancellationToken).ConfigureAwait(false); } heartbeatInfo = await GetHeartbeatInfoAsync(_heartbeatConnection, cancellationToken).ConfigureAwait(false); heartbeatException = null; break; } catch (Exception ex) { heartbeatException = ex; _heartbeatConnection.Dispose(); _heartbeatConnection = null; if (attempt == maxRetryCount) { _connectionPool.Clear(); } } } ServerDescription newDescription; if (heartbeatInfo != null) { var averageRoundTripTime = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime); var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds)); var isMasterResult = heartbeatInfo.IsMasterResult; var buildInfoResult = heartbeatInfo.BuildInfoResult; newDescription = _baseDescription.With( averageRoundTripTime: averageRoundTripTimeRounded, canonicalEndPoint: isMasterResult.Me, electionId: isMasterResult.ElectionId, maxBatchCount: isMasterResult.MaxBatchCount, maxDocumentSize: isMasterResult.MaxDocumentSize, maxMessageSize: isMasterResult.MaxMessageSize, replicaSetConfig: isMasterResult.GetReplicaSetConfig(), state: ServerState.Connected, tags: isMasterResult.Tags, type: isMasterResult.ServerType, version: buildInfoResult.ServerVersion, wireVersionRange: new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion)); } else { newDescription = _baseDescription; } if (heartbeatException != null) { newDescription = newDescription.With(heartbeatException: heartbeatException); } OnDescriptionChanged(newDescription); return(true); }
private void Heartbeat() { int pingFailCount = 0; logger.Info("HeartBeat Thread Started."); //heart-beat thread handles its own errors. try { HeartbeatInfo info = new HeartbeatInfo(); info.Interval = _HeartbeatInterval; // init for cpu usage TimeSpan oldTime = Process.GetCurrentProcess().TotalProcessorTime; DateTime timeMeasured = DateTime.Now; TimeSpan newTime = new TimeSpan(0); //TODO: be language neutral here. how?? // init for cpu avail PerformanceCounter cpuCounter = new PerformanceCounter(); cpuCounter.ReadOnly = true; cpuCounter.CategoryName = "Processor"; cpuCounter.CounterName = "% Processor Time"; cpuCounter.InstanceName = "_Total"; while (!_stop) { // calculate usage newTime = Process.GetCurrentProcess().TotalProcessorTime; TimeSpan absUsage = newTime - oldTime; float flUsage = ((float)absUsage.Ticks / (DateTime.Now - timeMeasured).Ticks) * 100; info.PercentUsedCpuPower = (int)flUsage > 100 ? 100 : (int)flUsage; info.PercentUsedCpuPower = (int)flUsage < 0 ? 0 : (int)flUsage; timeMeasured = DateTime.Now; oldTime = newTime; try { // calculate avail info.PercentAvailCpuPower = 100 - (int)cpuCounter.NextValue() + info.PercentUsedCpuPower; info.PercentAvailCpuPower = info.PercentAvailCpuPower > 100 ? 100 : info.PercentAvailCpuPower; info.PercentAvailCpuPower = info.PercentAvailCpuPower < 0 ? 0 : info.PercentAvailCpuPower; } catch (Exception e) { logger.Debug("HeartBeat thread error: " + e.Message + Environment.NewLine + " Heartbeat continuing after error..."); } //have a seperate try...block since we try 3 times before giving up try { //send a heartbeat to the manager. _executor.Manager.Executor_Heartbeat(_executor.Credentials, _executor.Id, info); pingFailCount = 0; } catch (Exception e) { if (e is SocketException || e is System.Runtime.Remoting.RemotingException) { pingFailCount++; //we disconnect the executor if the manager cant be pinged three times if (pingFailCount >= 3) { logger.Error("Failed to contact manager " + pingFailCount + " times...", e); //the disconnect here should be started off on a seperate thread because: //disconnect itself waits for HeartBeatThread to stop. If the method call //to disconnect from HeartBeat wont return immediately, then there is a deadlock //with disconnect waiting for the HeartBeatThread to stop and the HeartBeatThread waiting //for the call to disconnect to return. //raise the event to indicate that the Executor has got disconnected. _executor.OnGotDisconnected(); } } else if (e is InvalidExecutorException) { //raise a disconnected event. //so that we will reconnect if needed. logger.Error("Invalid Executor exception : ", e); _executor.OnGotDisconnected(); } else { logger.Debug("Error during heartbeat. Continuing after error...", e); } } Thread.Sleep(_HeartbeatInterval * 1000); } } catch (ThreadAbortException) { Thread.ResetAbort(); logger.Info("HeartBeat thread aborted."); } catch (Exception e) { logger.Error("HeartBeat Exception. Heartbeat thread stopping...", e); } logger.Info("HeartBeat Thread Exited."); }
private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken) { const int maxRetryCount = 2; HeartbeatInfo heartbeatInfo = null; Exception heartbeatException = null; for (var attempt = 1; attempt <= maxRetryCount; attempt++) { var connection = _connection; try { if (connection == null) { connection = _connectionFactory.CreateConnection(_serverId, _endPoint); // if we are cancelling, it's because the server has // been shut down and we really don't need to wait. await connection.OpenAsync(cancellationToken).ConfigureAwait(false); } heartbeatInfo = await GetHeartbeatInfoAsync(connection, cancellationToken).ConfigureAwait(false); heartbeatException = null; _connection = connection; break; } catch (Exception ex) { heartbeatException = ex; _connection = null; if (connection != null) { connection.Dispose(); } } } ServerDescription newDescription; if (heartbeatInfo != null) { var averageRoundTripTime = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime); var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds)); var isMasterResult = heartbeatInfo.IsMasterResult; var buildInfoResult = heartbeatInfo.BuildInfoResult; newDescription = _baseDescription.With( averageRoundTripTime: averageRoundTripTimeRounded, canonicalEndPoint: isMasterResult.Me, electionId: isMasterResult.ElectionId, lastWriteTimestamp: isMasterResult.LastWriteTimestamp, logicalSessionTimeout: isMasterResult.LogicalSessionTimeout, maxBatchCount: isMasterResult.MaxBatchCount, maxDocumentSize: isMasterResult.MaxDocumentSize, maxMessageSize: isMasterResult.MaxMessageSize, replicaSetConfig: isMasterResult.GetReplicaSetConfig(), state: ServerState.Connected, tags: isMasterResult.Tags, topologyVersion: isMasterResult.TopologyVersion, type: isMasterResult.ServerType, version: buildInfoResult.ServerVersion, wireVersionRange: new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion)); } else { newDescription = _baseDescription.With(lastUpdateTimestamp: DateTime.UtcNow); } if (heartbeatException != null) { var topologyVersion = default(Optional <TopologyVersion>); if (heartbeatException is MongoCommandException heartbeatCommandException) { topologyVersion = TopologyVersion.FromMongoCommandException(heartbeatCommandException); } newDescription = newDescription.With(heartbeatException: heartbeatException, topologyVersion: topologyVersion); } newDescription = newDescription.With(reasonChanged: "Heartbeat", lastHeartbeatTimestamp: DateTime.UtcNow); SetDescription(newDescription); return(true); }
protected VariableParam readVariable() { int type = _body.GetUShort(); int len = _body.GetUShort(); int blen = len - 4; //byte[] data; Unknown var; switch (type) { case 1: var = new HeartbeatInfo(1, "HeartbeatInfo"); break; // 2-4 Unassigned case 5: var = new IPv4Address(5, "IPv4Address"); break; case 6: var = new IPv6Address(6, "IPv6Address"); break; case 7: var = new StateCookie(7, "StateCookie"); break; case 8: var = new UnrecognizedParameters(8, "UnrecognizedParameters"); break; case 9: var = new CookiePreservative(9, "CookiePreservative"); break; // 10 Unassigned case 11: var = new HostNameAddress(11, "HostNameAddress"); break; case 12: var = new SupportedAddressTypes(12, "SupportedAddressTypes"); break; case 13: var = new OutgoingSSNResetRequestParameter(13, "OutgoingSSNResetRequestParameter"); break; case 14: var = new IncomingSSNResetRequestParameter(14, "IncomingSSNResetRequestParameter"); break; case 15: var = new SSNTSNResetRequestParameter(15, "SSNTSNResetRequestParameter"); break; case 16: var = new ReconfigurationResponseParameter(16, "ReconfigurationResponseParameter"); break; case 17: var = new AddOutgoingStreamsRequestParameter(17, "AddOutgoingStreamsRequestParameter"); break; case 18: var = new AddIncomingStreamsRequestParameter(18, "AddIncomingStreamsRequestParameter"); break; // 19-32767 Unassigned case 32768: var = new Unknown(32768, "ReservedforECNCapable"); break; case 32770: var = new RandomParam(32770, "Random"); break; case 32771: var = new ChunkListParam(32771, "ChunkList"); break; case 32772: var = new RequestedHMACAlgorithmParameter(32772, "RequestedHMACAlgorithmParameter"); break; case 32773: var = new Unknown(32773, "Padding"); break; case 32776: var = new SupportedExtensions(32776, "SupportedExtensions"); break; // 32777-49151 Unassigned case 49152: var = new ForwardTSNsupported(49152, "ForwardTSNsupported"); break; case 49153: var = new Unknown(49153, "AddIPAddress"); break; case 49154: var = new Unknown(49154, "DeleteIPAddress"); break; case 49155: var = new Unknown(49155, "ErrorCauseIndication"); break; case 49156: var = new Unknown(49156, "SetPrimaryAddress"); break; case 49157: var = new Unknown(49157, "SuccessIndication"); break; case 49158: var = new Unknown(49158, "AdaptationLayerIndication"); break; default: var = new Unknown(-1, "Unknown"); break; } try { var.readBody(_body, blen); //logger.LogDebug("variable type " + var.getType() + " name " + var.getName()); } catch (SctpPacketFormatException ex) { logger.LogError(ex.ToString()); } if (_body.hasRemaining()) { int mod = blen % 4; if (mod != 0) { for (int pad = mod; pad < 4; pad++) { _body.GetByte(); } } } return(var); }
internal ElectionVote CastVote(Election election, Membership existingMembership, HeartbeatInfo info) { if (election == null || election.ElectionId == null) { if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled) { LoggerManager.Instance.ShardLogger.Warn("ElectionManager.CastVote()", "Invalid election."); } return(null); } ElectionVote vote = new ElectionVote(); vote.RequestingNode = election.ElectionId.RequestingNode; vote.Sourcenode = new ServerNode(); vote.Sourcenode.Name = _context.LocalAddress.IpAddress.ToString(); vote.Sourcenode.Priority = (int)LocalServerPriority; if (existingMembership != null && (election.ElectionType == ElectionType.GeneralElections && existingMembership.Primary == null) || election.ElectionType == ElectionType.TakeoverElections) { if (election.RequestingServerInfo.LastOperationId > LocalServerInfo().LastOperationId) { vote.NodeVote = ElectionVote.Vote.yes; } else if (election.RequestingServerInfo.LastOperationId == LocalServerInfo().LastOperationId) { if (election.ElectionId.RequestingNode.Priority <= (int)LocalServerPriority) { vote.NodeVote = ElectionVote.Vote.yes; } else { ShardConfiguration sConfig = null; if (_clusterConfigMgr != null) { sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName); } int configuredNodes = 0; if (sConfig != null || sConfig.Servers != null) { configuredNodes = sConfig.Servers.Nodes.Count; } if (info != null && info.CSStatus == ConnectivityStatus.Connected && (configuredNodes != null && existingMembership.Servers.Count >= configuredNodes / 2)) { vote.NodeVote = ElectionVote.Vote.no; } else { vote.NodeVote = ElectionVote.Vote.yes; } } } else { vote.NodeVote = ElectionVote.Vote.no; } } else { vote.NodeVote = ElectionVote.Vote.no; } return(vote); }
/// <summary> /// Speculative Phase: /// 1. We search for the node with the latest op-log entry. This detail bears the highest value. /// 2. Next, if multiple nodes have the same op-log entry, we move onto the next step. /// 3. All those nodes which have the same op-log entry AND are connected to the CS are considered. /// 4. Highest priority from amongst these active nodes are taken into consideration. /// 5. If this node fulfills all of the above, it successfully passes the speculative phase. /// </summary> /// <param name="heartbeatReport"></param> /// <returns></returns> private bool ShouldIInitiateElection(LocalShardHeartbeatReporting heartbeatReport, Activity activity) { IList <Address> activeNodes = null; if (heartbeatReport != null) { activeNodes = heartbeatReport.GetReportTable.Keys.ToList(); OperationId maxOplog = null; IList <string> matchingOplogServerIPs = new List <string>(); HeartbeatInfo localHeartbeat = heartbeatReport.GetHeartbeatInfo(_context.LocalAddress); if (localHeartbeat == null) { if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled) { LoggerManager.Instance.ShardLogger.Warn( "ElectionBasedMembershipStrategy.ShouldIInititateElections()", "local node heartbeat is null"); } return(false); } OperationId lastRepId = null; if (activity.Equals(Activity.TakeoverElectionsTriggered)) { HeartbeatInfo info = null; if (_shard != null && _shard.Primary != null) { info = heartbeatReport.GetHeartbeatInfo(_shard.Primary.Address); } } else { ShardConfiguration sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName); int configuredNodesCount = 0; OperationId OpIdAtCS = null; if (sConfig != null && sConfig.Servers != null && sConfig.Servers.Nodes != null) { configuredNodesCount = sConfig.Servers.Nodes.Count; } if (configuredNodesCount > 0 && activeNodes != null && activeNodes.Count < configuredNodesCount) { ShardInfo sInfo = null; ClusterInfo cInfo = _context.ConfigurationSession.GetDatabaseClusterInfo(_context.ClusterName); if (cInfo != null) { sInfo = cInfo.GetShardInfo(_context.LocalShardName); } if (sInfo != null) { OpIdAtCS = sInfo.LastOperationId; } if (OpIdAtCS > lastRepId) { if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled) { LoggerManager.Instance.ShardLogger.Info( "electBasedMemSt.ShouldIInitElections()", "CS has an operation newer than my operation. Hence, waiting."); } if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled) { if (OpIdAtCS != null) { LoggerManager.Instance.ShardLogger.Info( "electBasedMemSt.ShouldIInitElections()", "Operation ID on the CS:- " + OpIdAtCS.ElectionId + ":" + OpIdAtCS.ElectionBasedSequenceId); } else { LoggerManager.Instance.ShardLogger.Info( "electBasedMemSt.ShouldIInitElections()", "The operation ID at the CS is set to null."); } if (lastRepId != null) { LoggerManager.Instance.ShardLogger.Info( "electBasedMemSt.ShouldIInitElections()", "Local node Operation ID:- " + lastRepId.ElectionId + ":" + lastRepId.ElectionBasedSequenceId); } else { LoggerManager.Instance.ShardLogger.Info( "electBasedMemSt.ShouldIInitElections()", "The local node operation ID is set to null."); } } //We maintain the last replicated operation log entry with the CS. //If a node in a shard with older data (usually the previous secondary) is up before the node with the //latest data(usually the previous primary), it waits for a configurable amount of time (2 minutes for //now) before proceeding with the election procedure if it is still unable to detect a primary node. //This way we give the node with the latest data a chance to become primary and therefore avoid data loss. lock (_mutexOnWait) { Monitor.Wait(_mutexOnWait, _waitTimeout); } } } for (int i = 0; i < activeNodes.Count; i++) { HeartbeatInfo info = heartbeatReport.GetHeartbeatInfo(activeNodes[i]); OperationId currIndexOplog = info.LastOplogOperationId; if (currIndexOplog > maxOplog) { maxOplog = currIndexOplog; } if (((localHeartbeat.LastOplogOperationId == null && info.LastOplogOperationId == null) || localHeartbeat.LastOplogOperationId != null && localHeartbeat.LastOplogOperationId.Equals(info.LastOplogOperationId)) && info.CSStatus == ConnectivityStatus.Connected) { matchingOplogServerIPs.Add(activeNodes[i].IpAddress.ToString()); } } } if (localHeartbeat.LastOplogOperationId != null && maxOplog != null && maxOplog > localHeartbeat.LastOplogOperationId && (lastRepId == null || maxOplog > lastRepId)) { if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "Local operation log is behind max op log wrt " + _context.LocalShardName + " shard."); } if (maxOplog != null && LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "maxOplog: " + maxOplog.ElectionId + ":" + maxOplog.ElectionBasedSequenceId); } if (localHeartbeat.LastOplogOperationId != null && LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "local opLog (from the heartbeat): " + localHeartbeat.LastOplogOperationId.ElectionId + ":" + localHeartbeat.LastOplogOperationId.ElectionBasedSequenceId); } if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled && lastRepId != null) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "LastOpFromOpLog (from the replication module): " + lastRepId.ElectionId + ":" + lastRepId.ElectionBasedSequenceId); } return(false); } else if (maxOplog == localHeartbeat.LastOplogOperationId || (lastRepId != null && lastRepId.Equals(maxOplog))) { //if: there are multiple nodes that have the same oplog entry, //decision will be made on the basis of the priorities. //else: the node with the highest oplog entry will be considered eligible. if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "Local operation log is equal to the max op log wrt " + _context.LocalShardName + " shard."); } if (maxOplog != null && (localHeartbeat != null && localHeartbeat.LastOplogOperationId != null)) { if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "maxOplog: " + maxOplog.ElectionId + ":" + maxOplog.ElectionBasedSequenceId); } if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "local opLog (from the heartbeat): " + localHeartbeat.LastOplogOperationId.ElectionId + ":" + localHeartbeat.LastOplogOperationId.ElectionBasedSequenceId); } if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled && lastRepId != null) { LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "LastOpFromOpLog (from the replication module): " + lastRepId.ElectionId + ":" + lastRepId.ElectionBasedSequenceId); } } if (matchingOplogServerIPs.Count > 0) { int highestRunningNodePriority = _manager.GetHighestNodePriority(activeNodes, matchingOplogServerIPs); if (highestRunningNodePriority.Equals(_manager.LocalServerPriority)) { if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled) { LoggerManager.Instance.ShardLogger.Debug( "ElectionBasedMembershipStrategy.ShouldIInitiateElection()", "Node : " + _context.LocalAddress.IpAddress.ToString() + " in shard: " + _context.LocalShardName + " is eligible having priority: " + highestRunningNodePriority + " ."); } return(true); } else { return(false); } } } } return(false); }
private void ManageHeartbeats() { if (_connection == null) throw new Exception(); var timeout = _heartbeatInfo.IsIntervalStage ? _settings.HeartbeatInterval : _settings.HeartbeatTimeout; if (_stopwatch.Elapsed - _heartbeatInfo.TimeStamp < timeout) return; var packageNumber = _packageNumber; if (_heartbeatInfo.LastPackageNumber != packageNumber) { _heartbeatInfo = new HeartbeatInfo(packageNumber, true, _stopwatch.Elapsed); return; } if (_heartbeatInfo.IsIntervalStage) { // TcpMessage.Heartbeat analog _connection.EnqueueSend(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null)); _heartbeatInfo = new HeartbeatInfo(_heartbeatInfo.LastPackageNumber, false, _stopwatch.Elapsed); } else { // TcpMessage.HeartbeatTimeout analog var msg = string.Format("EventStoreConnection '{0}': closing TCP connection [{1}, L{2}, {3:B}] due to HEARTBEAT TIMEOUT at pkgNum {4}.", _esConnection.ConnectionName, _connection.RemoteEndPoint, _connection.LocalEndPoint, _connection.ConnectionId, packageNumber); _settings.Log.Info(msg); CloseTcpConnection(msg); } }
public Task Heartbeat(HeartbeatInfo heartbeat) { return(Task.Factory.StartNew(() => HeartbeatService.Heartbeat(heartbeat))); }