private IList <Address> CheckForLostNodes(IDictionary <Address, HeartbeatInfo> reportTable)
{
IList <Address> lostNodes = new List <Address>();
IList <Address> serverList = reportTable.Keys.ToList();
if (serverList != null)
{
for (int index = 0; index < serverList.Count; index++)
{
HeartbeatInfo serverInfo = reportTable[serverList[index]];
lock (serverInfo)
{
if (serverInfo != null && serverInfo.MissingHeartbeatsCounter >= _maxNumberOfMissingHeartbeats)
{
lostNodes.Add(serverList[index]);
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartbeatTask.CheckForLostNodes() ", serverInfo.MissingHeartbeatsCounter.ToString() + " heartbeats from node " + serverList[index].IpAddress.ToString() + " missed.");
}
}
}
}
}
return(lostNodes);
}
/// <summary>
/// This methods checks the heartbeats with the current table.
/// </summary>
/// <returns></returns>
private void CheckHeartbeats(IDictionary <Address, HeartbeatInfo> reportTable)
{
if (reportTable != null)
{
IList <Address> serverList = reportTable.Keys.ToList();
if (serverList != null)
{
for (int index = 0; index < serverList.Count; index++)
{
if (reportTable.ContainsKey(serverList[index]))
{
HeartbeatInfo serverInfo = reportTable[serverList[index]];
lock (serverInfo)
{
if (serverInfo != null)
{
DateTime?lastHeartbeat = serverInfo.LastHeartbeatTimestamp;
if (lastHeartbeat.HasValue)
{
if (lastHeartbeat.Value.AddSeconds(_maxInterval) < DateTime.Now)
{
_localShardHeartbeatReporter.UpdateMissingHeartbeatsCount(serverList[index]);
}
}
}
}
}
}
}
}
}
internal bool PrimaryExists()
{
IList <Address> activeNodes = _heartbeatReporter.Keys.ToList();
if (activeNodes != null)
{
foreach (var node in activeNodes)
{
HeartbeatInfo info = GetHeartbeatInfo(node);
lock (info)
{
if (info != null && info.CurrentMembership != null && info.CurrentMembership.Primary != null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartReporting.PrimaryExists() ", "Primary for this shard is: " + node.IpAddress.ToString());
}
return(true);
}
}
}
}
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartReporting.PrimaryExists() ", "Primary for this shard does not exist. ");
}
return(false);
}
internal ElectionId GetCurrentElectionId()
{
if (PrimaryExists())
{
ServerNode primary = GetCurrentPrimary();
IList <Address> activeNodes = _heartbeatReporter.Keys.ToList();
if (activeNodes != null)
{
foreach (var node in activeNodes)
{
if (node.IpAddress.ToString() == primary.Name)
{
HeartbeatInfo info = GetHeartbeatInfo(node);
lock (info)
{
if (info != null && info.CurrentMembership != null && info.CurrentMembership.ElectionId != null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartReporting.GetCurrentEditionId() ", "Election ID of the ongoing election term is " + info.CurrentMembership.ElectionId.Id);
}
return(info.CurrentMembership.ElectionId);
}
}
}
}
}
}
return(null);
}
internal ServerNode GetCurrentPrimary()
{
if (_heartbeatReporter != null && PrimaryExists())
{
IList <Address> activeNodes = _heartbeatReporter.Keys.ToList();
if (activeNodes != null)
{
foreach (var node in activeNodes)
{
HeartbeatInfo info = GetHeartbeatInfo(node);
lock (info)
{
if (info != null && info.CurrentMembership != null && info.CurrentMembership.Primary != null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartReporting.GetCurrentPrimary() ", "Primary for this shard is: " + info.CurrentMembership.Primary.Name.ToString());
}
return(info.CurrentMembership.Primary);
}
}
}
}
}
return(null);
}
// Receives the heartbeats from different nodes which are part of a shard and
// adds them to the report table.
public void ReceiveHeartbeat(Address source, HeartbeatInfo heartbeatInfo)
{
try
{
if (_localShardHeartbeatReporter != null)
{
// We need to verify if the node sending the heartbeat is part of the existing configuration.
// Updating the config is a costly process so we check the node in the existing config.
// If the node was freshly added in an existing (active) shard, we will not add the heartbeat to the report
// until it exists in the local node config instance.
if (_clusterConfigMgr != null)
{
ShardConfiguration sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
ServerNode sNode = null;
if (sConfig != null && sConfig.Servers != null)
{
sNode = sConfig.Servers.GetServerNode(source.IpAddress.ToString());
}
if (sNode == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartbeatTask.ReceiveHeartbeat() ", "The node " + source + " is not part of the configuration.");
}
return;
}
}
bool isAnOldNode = _localShardHeartbeatReporter.AddToReport(source, heartbeatInfo);
if (!isAnOldNode)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartbeatTask.ReceiveHeartbeat() ", "Node " + source.IpAddress + " added to the table for the first time. ");
}
lock (_membershipManager)
{
_membershipManager.HeartbeatReport = _localShardHeartbeatReporter;
}
_electionExecTask = new ElectionMechanismExecutionTask(_membershipManager, Activity.NodeJoining, new Server(source, Status.Running));
_electionExecTask.Start();
OnActivityCompleted();
}
}
}
catch (Exception e)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("LocalShardCheckHeartbeatTask.ReceiveHeartbeat() ", e.ToString());
}
}
}
public object Clone()
{
HeartbeatInfo nodeInfo = new HeartbeatInfo();
nodeInfo.CurrentMembership = CurrentMembership;
nodeInfo.LastOplogOperationId = LastOplogOperationId;
nodeInfo.MissingHeartbeatsCounter = MissingHeartbeatsCounter;
nodeInfo.LastOplogOperationId = LastOplogOperationId;
nodeInfo.CSStatus = CSStatus;
return(nodeInfo);
}
/// <summary>
/// Whenever a new heartbeat is received, it is updated in the heartbeat table.
/// </summary>
/// <param name="source"></param>
/// <param name="info"></param>
public void ReceiveHeartbeat(Address source, HeartbeatInfo info)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("HeatbeatManager.ReceiveHeartbeat() ", "Heartbeat from node " + source.IpAddress.ToString() + " received at " + DateTime.Now.ToString());
}
if (_checkHeartbeatTask != null)
{
_checkHeartbeatTask.ReceiveHeartbeat(source, info);
}
}
// The method which runs in a thread.
public void Run()
{
LoggerManager.Instance.SetThreadContext(new LoggerContext()
{
ShardName = _nodeContext.LocalShardName != null ? _nodeContext.LocalShardName : "", DatabaseName = ""
});
_nodeContext.StatusLatch.WaitForAny(NodeStatus.Running);
_startSignal.WaitOne();
while (_running)
{
try
{
// 1. update the local node information
HeartbeatInfo info = UpdateLocalNodeData();
if (info != null)
{
// 2. Send the heartbeat info to the cluster.
SendHeartbeat(info);
if (LoggerManager.Instance.ShardLogger != null &&
LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("LocalShardCheckHeartbeatTask.Run() ", "Heartbeat broadcasted from node " + _nodeContext.LocalAddress.IpAddress.ToString() + " at " + DateTime.Now.ToString());
}
}
}
catch (ThreadAbortException e)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled && _sendHbThread != null)
{
LoggerManager.Instance.ShardLogger.Error(_sendHbThread.Name, "Task aborted.");
}
break;
}
catch (ChannelException ex)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("LocalShardSendHeartbeatTask.Run() ", ex.ToString());
}
}
catch (Exception ex)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsErrorEnabled)
{
LoggerManager.Instance.ShardLogger.Error("LocalShardSendHeartbeatTask.Run() " + ex.ToString());
}
}
Thread.Sleep(_poolingThreshold);
_startSignal.WaitOne();
}
}
//RTD: revisit this logic
public object Clone()
{
LocalShardHeartbeatReporting clone = new LocalShardHeartbeatReporting(_minReplicatedOperation.Key);
clone._heartbeatReporter = new Dictionary <Address, HeartbeatInfo>();
IList <Address> nodes = this._heartbeatReporter.Keys.ToList();
foreach (var node in nodes)
{
HeartbeatInfo info = _heartbeatReporter[node];
lock (info)
{
clone._heartbeatReporter.Add(node, (HeartbeatInfo)info.Clone());
}
}
return(clone);
}
internal HeartbeatInfo UpdateLocalNodeData()
{
HeartbeatInfo info = new HeartbeatInfo();
if (_nodeContext != null)
{
info.CurrentMembership = _membershipManager.LatestMembership;
info.LastOplogOperationId = null;
_membershipManager.UpdateLastOperationId(info.LastOplogOperationId);
info.CSStatus = _membershipManager.CSStatus;
}
return(info);
}
internal void SendHeartbeat(HeartbeatInfo info)
{
List <Server> list = new List <Server>();
if (_shard != null && _shard.ActiveChannelsList != null)
{
Message msg = new Message();
msg.MessageType = MessageType.Heartbeat;
msg.Payload = info;
msg.NeedsResponse = false;
ShardMulticastRequest <ResponseCollection <object>, object> request = _shard.CreateMulticastRequest <ResponseCollection <object>, object>(_shard.ActiveChannelsList, msg);
IAsyncResult result = request.BeginExecute();
ResponseCollection <Object> response = request.EndExecute(result);
}
}
public bool AddToReport(Address source, HeartbeatInfo hbInfo)
{
bool isAnOldNode = false;
if (source != null)
{
// update the timestamp whenever a heartbeat is received.
hbInfo.LastHeartbeatTimestamp = DateTime.Now;
// Reset the missing heartbeats counter as soon as a heartbeat is received.
hbInfo.MissingHeartbeatsCounter = 0;
lock (_heartbeatReporter)
{
if (_heartbeatReporter.ContainsKey(source))
{
isAnOldNode = true;
}
else
{
if (LoggerManager.Instance.ShardLogger != null &&
LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info("LocalShardHeartbeatReporting.AddToReport() ",
"Node joining activity triggered: " + source.IpAddress.ToString() +
" added to the report table.");
}
}
_heartbeatReporter[source] = hbInfo;
}
KeyValuePair <Address, OperationId> lastReplicatedOperation = new KeyValuePair <Address, OperationId>(source, hbInfo.LastOplogOperationId);
lock (_mutex)
{
if (_minReplicatedOperation.Key.Equals(source))
{
_minReplicatedOperation = lastReplicatedOperation;
}
else if (_minReplicatedOperation.Value > hbInfo.LastOplogOperationId)
{
_minReplicatedOperation = lastReplicatedOperation;
}
}
}
return(isAnOldNode);
}