private void TcpConnectionEstablished(TcpPackageConnection connection)
{
if (_state != ConnectionState.Connecting || _connection != connection || connection.IsClosed)
{
LogDebug("IGNORED (_state {0}, _conn.Id {1:B}, conn.Id {2:B}, conn.closed {3}): TCP connection to [{4}, L{5}] established.",
_state, _connection == null ? Guid.Empty : _connection.ConnectionId, connection.ConnectionId,
connection.IsClosed, connection.RemoteEndPoint, connection.LocalEndPoint);
return;
}
LogDebug("TCP connection to [{0}, L{1}, {2:B}] established.", connection.RemoteEndPoint, connection.LocalEndPoint, connection.ConnectionId);
_heartbeatInfo = new HeartbeatInfo(_packageNumber, true, _stopwatch.Elapsed);
if (_settings.DefaultUserCredentials != null)
{
_connectingPhase = ConnectingPhase.Authentication;
_authInfo = new AuthInfo(Guid.NewGuid(), _stopwatch.Elapsed);
_connection.EnqueueSend(new TcpPackage(TcpCommand.Authenticate,
TcpFlags.Authenticated,
_authInfo.CorrelationId,
_settings.DefaultUserCredentials.Username,
_settings.DefaultUserCredentials.Password,
null));
}
else
{
GoToConnectedState();
}
}
public HeartbeatConfigFromServer Info(HeartbeatInfo info)
{
tracker.OnInfo(info);
return(new HeartbeatConfigFromServer
{
SendAtIntervals = true,
SendChanges = true,
});
}
private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken)
{
const int maxRetryCount = 2;
HeartbeatInfo heartbeatInfo = null;
for (var attempt = 1; attempt <= maxRetryCount; attempt++)
{
try
{
if (_heartbeatConnection == null)
{
_heartbeatConnection = _heartbeatConnectionFactory.CreateConnection(_serverId, _endPoint);
await _heartbeatConnection.OpenAsync(TimeSpan.FromMinutes(1), cancellationToken);
}
heartbeatInfo = await GetHeartbeatInfoAsync(_heartbeatConnection, cancellationToken);
break;
}
catch
{
_heartbeatConnection.Dispose();
_heartbeatConnection = null;
if (attempt == maxRetryCount)
{
_connectionPool.Clear();
}
}
}
ServerDescription newDescription;
if (heartbeatInfo != null)
{
var averageRoundTripTime = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime);
var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds));
var isMasterResult = heartbeatInfo.IsMasterResult;
var buildInfoResult = heartbeatInfo.BuildInfoResult;
newDescription = _baseDescription.WithHeartbeatInfo(
averageRoundTripTimeRounded,
isMasterResult.GetReplicaSetConfig(),
isMasterResult.Tags,
isMasterResult.ServerType,
buildInfoResult.ServerVersion,
new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion));
}
else
{
newDescription = _baseDescription;
}
OnDescriptionChanged(newDescription);
return(true);
}
public void Heartbeat(HeartbeatInfo heartbeat)
{
try
{
Trace.TraceInformation(JsonConvert.SerializeObject(heartbeat));
}
catch
{
}
}
/// <summary>
/// Updates the database with the heartbeat info of this executor
/// </summary>
/// <param name="info"></param>
public void HeartbeatUpdate(HeartbeatInfo info)
{
// update ping time and other heartbeatinfo
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
executorStorage.PingTime = DateTime.Now;
executorStorage.Connected = true;
executorStorage.CpuUsage = info.PercentUsedCpuPower;
executorStorage.AvailableCpu = info.PercentAvailCpuPower;
executorStorage.TotalCpuUsage += info.Interval * (float)info.PercentUsedCpuPower / 100;
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
}
public object OnMessageReceived(object message, Server source, LocalShardHeartbeatReporting heartbeatReport, Membership existingMembership)
{
Election election = message as Election;
if (election != null)
{
HeartbeatInfo localHeartbeat = heartbeatReport.GetHeartbeatInfo(_context.LocalAddress);
if (localHeartbeat != null)
{
return(_manager.CastVote(election, existingMembership, localHeartbeat));
}
}
return(null);
}
public HeartbeatInfo GetPlatformHeartBeatInfo()
{
HeartbeatInfo hbi = new HeartbeatInfo();
hbi.HomeId = Settings.HomeId;
hbi.OrgId = Settings.OrgId;
hbi.StudyId = Settings.StudyId;
hbi.HubTimestamp = DateTime.UtcNow.ToString();
hbi.PhysicalMemoryBytes = this.perfCountWorkingSet.NextValue();
hbi.TotalCpuPercentage = this.perfCountPercentProcTime.NextValue();
hbi.ModuleMonitorInfoList = this.platform.GetModuleMonitorInfoList();
hbi.HeartbeatIntervalMins = this.heartbeatIntervalMins;
hbi.SequenceNumber = this.sequenceNumber++;
hbi.HardwareId = HomeOS.Hub.Common.Utils.HardwareId;
return hbi;
}
public HeartbeatInfo GetPlatformHeartBeatInfo()
{
HeartbeatInfo hbi = new HeartbeatInfo();
hbi.HomeId = Settings.HomeId;
hbi.OrgId = Settings.OrgId;
hbi.StudyId = Settings.StudyId;
hbi.HubTimestamp = DateTime.UtcNow.ToString();
hbi.PhysicalMemoryBytes = this.perfCountWorkingSet.NextValue();
hbi.TotalCpuPercentage = this.perfCountPercentProcTime.NextValue();
hbi.ModuleMonitorInfoList = this.platform.GetModuleMonitorInfoList();
hbi.ScoutInfoList = this.platform.GetScoutInfoList();
hbi.HeartbeatIntervalMins = this.heartbeatIntervalMins;
hbi.SequenceNumber = this.sequenceNumber++;
hbi.HardwareId = HomeOS.Hub.Common.Utils.HardwareId;
hbi.PlatformVersion = this.platform.GetPlatformVersion();
return(hbi);
}
private void SendHeartbeat(Object stateInfo)
{
try
{
HeartbeatInfo heartbeatInfo = GetPlatformHeartBeatInfo();
if (null != heartbeatInfo)
{
string jsonString = heartbeatInfo.SerializeToJsonStream();
logger.Log("Sending heartbeat: {0}", jsonString);
var webClient = new NoKeepAlivesWebClient();
webClient.UploadStringCompleted += webClient_UploadHeartbeatInfoCompleted;
webClient.Headers["Content-type"] = "application/json";
webClient.Encoding = Encoding.UTF8;
webClient.UseDefaultCredentials = true;
webClient.UploadStringAsync(new Uri(this.uri.OriginalString + "/SetHeartbeatInfo"), "POST", jsonString, jsonString);
}
}
catch (Exception e)
{
logger.Log("Failed to send heartbeat, exception={0}", e.Message);
}
}
private void ManageHeartbeats()
{
if (_connection == null)
{
throw new Exception();
}
var timeout = _heartbeatInfo.IsIntervalStage ? _settings.HeartbeatInterval : _settings.HeartbeatTimeout;
if (_stopwatch.Elapsed - _heartbeatInfo.TimeStamp < timeout)
{
return;
}
var packageNumber = _packageNumber;
if (_heartbeatInfo.LastPackageNumber != packageNumber)
{
_heartbeatInfo = new HeartbeatInfo(packageNumber, true, _stopwatch.Elapsed);
return;
}
if (_heartbeatInfo.IsIntervalStage)
{
// TcpMessage.Heartbeat analog
_connection.EnqueueSend(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null));
_heartbeatInfo = new HeartbeatInfo(_heartbeatInfo.LastPackageNumber, false, _stopwatch.Elapsed);
}
else
{
// TcpMessage.HeartbeatTimeout analog
var msg = string.Format(
"EventStoreConnection '{0}': closing TCP connection [{1}, {2}, {3}] due to HEARTBEAT TIMEOUT at pkgNum {4}.",
_esConnection.ConnectionName, _connection.RemoteEndPoint, _connection.LocalEndPoint,
_connection.ConnectionId, packageNumber);
_settings.Log.Info(msg);
CloseTcpConnection(msg);
}
}
public void SetHeartbeatInfo(HeartbeatInfo hbi)
{
if (null == hbi.OrgId || hbi.OrgId.Length == 0)
{
hbi.OrgId = "Unknown";
}
HeartbeatTable HeartbeatTable = Helper.GetHeartbeatTable();
if (null == hbi)
{
throw new ArgumentNullException();
}
try
{
HeartbeatTable.Write(hbi.OrgId, hbi.HomeId, hbi.SerializeToJsonStream());
}
catch(Exception e)
{
Helper.Trace().WriteLine("SimpleStorage.Write Failed with Exception:{0}", e.Message);
}
}
private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken)
{
const int maxRetryCount = 2;
HeartbeatInfo heartbeatInfo = null;
Exception heartbeatException = null;
for (var attempt = 1; attempt <= maxRetryCount; attempt++)
{
try
{
if (_heartbeatConnection == null)
{
_heartbeatConnection = _heartbeatConnectionFactory.CreateConnection(_serverId, _endPoint);
// if we are cancelling, it's because the server has
// been shut down and we really don't need to wait.
await _heartbeatConnection.OpenAsync(cancellationToken).ConfigureAwait(false);
}
heartbeatInfo = await GetHeartbeatInfoAsync(_heartbeatConnection, cancellationToken).ConfigureAwait(false);
heartbeatException = null;
break;
}
catch (Exception ex)
{
heartbeatException = ex;
_heartbeatConnection.Dispose();
_heartbeatConnection = null;
if (attempt == maxRetryCount)
{
_connectionPool.Clear();
}
}
}
ServerDescription newDescription;
if (heartbeatInfo != null)
{
var averageRoundTripTime = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime);
var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds));
var isMasterResult = heartbeatInfo.IsMasterResult;
var buildInfoResult = heartbeatInfo.BuildInfoResult;
newDescription = _baseDescription.With(
averageRoundTripTime: averageRoundTripTimeRounded,
canonicalEndPoint: isMasterResult.Me,
electionId: isMasterResult.ElectionId,
maxBatchCount: isMasterResult.MaxBatchCount,
maxDocumentSize: isMasterResult.MaxDocumentSize,
maxMessageSize: isMasterResult.MaxMessageSize,
replicaSetConfig: isMasterResult.GetReplicaSetConfig(),
state: ServerState.Connected,
tags: isMasterResult.Tags,
type: isMasterResult.ServerType,
version: buildInfoResult.ServerVersion,
wireVersionRange: new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion));
}
else
{
newDescription = _baseDescription;
}
if (heartbeatException != null)
{
newDescription = newDescription.With(heartbeatException: heartbeatException);
}
OnDescriptionChanged(newDescription);
return(true);
}
private void Heartbeat()
{
int pingFailCount = 0;
logger.Info("HeartBeat Thread Started.");
//heart-beat thread handles its own errors.
try
{
HeartbeatInfo info = new HeartbeatInfo();
info.Interval = _HeartbeatInterval;
// init for cpu usage
TimeSpan oldTime = Process.GetCurrentProcess().TotalProcessorTime;
DateTime timeMeasured = DateTime.Now;
TimeSpan newTime = new TimeSpan(0);
//TODO: be language neutral here. how??
// init for cpu avail
PerformanceCounter cpuCounter = new PerformanceCounter();
cpuCounter.ReadOnly = true;
cpuCounter.CategoryName = "Processor";
cpuCounter.CounterName = "% Processor Time";
cpuCounter.InstanceName = "_Total";
while (!_stop)
{
// calculate usage
newTime = Process.GetCurrentProcess().TotalProcessorTime;
TimeSpan absUsage = newTime - oldTime;
float flUsage = ((float)absUsage.Ticks / (DateTime.Now - timeMeasured).Ticks) * 100;
info.PercentUsedCpuPower = (int)flUsage > 100 ? 100 : (int)flUsage;
info.PercentUsedCpuPower = (int)flUsage < 0 ? 0 : (int)flUsage;
timeMeasured = DateTime.Now;
oldTime = newTime;
try
{
// calculate avail
info.PercentAvailCpuPower = 100 - (int)cpuCounter.NextValue() + info.PercentUsedCpuPower;
info.PercentAvailCpuPower = info.PercentAvailCpuPower > 100 ? 100 : info.PercentAvailCpuPower;
info.PercentAvailCpuPower = info.PercentAvailCpuPower < 0 ? 0 : info.PercentAvailCpuPower;
}
catch (Exception e)
{
logger.Debug("HeartBeat thread error: " + e.Message + Environment.NewLine + " Heartbeat continuing after error...");
}
//have a seperate try...block since we try 3 times before giving up
try
{
//send a heartbeat to the manager.
_executor.Manager.Executor_Heartbeat(_executor.Credentials, _executor.Id, info);
pingFailCount = 0;
}
catch (Exception e)
{
if (e is SocketException || e is System.Runtime.Remoting.RemotingException)
{
pingFailCount++;
//we disconnect the executor if the manager cant be pinged three times
if (pingFailCount >= 3)
{
logger.Error("Failed to contact manager " + pingFailCount + " times...", e);
//the disconnect here should be started off on a seperate thread because:
//disconnect itself waits for HeartBeatThread to stop. If the method call
//to disconnect from HeartBeat wont return immediately, then there is a deadlock
//with disconnect waiting for the HeartBeatThread to stop and the HeartBeatThread waiting
//for the call to disconnect to return.
//raise the event to indicate that the Executor has got disconnected.
_executor.OnGotDisconnected();
}
}
else if (e is InvalidExecutorException)
{
//raise a disconnected event.
//so that we will reconnect if needed.
logger.Error("Invalid Executor exception : ", e);
_executor.OnGotDisconnected();
}
else
{
logger.Debug("Error during heartbeat. Continuing after error...", e);
}
}
Thread.Sleep(_HeartbeatInterval * 1000);
}
}
catch (ThreadAbortException)
{
Thread.ResetAbort();
logger.Info("HeartBeat thread aborted.");
}
catch (Exception e)
{
logger.Error("HeartBeat Exception. Heartbeat thread stopping...", e);
}
logger.Info("HeartBeat Thread Exited.");
}
private async Task <bool> HeartbeatAsync(CancellationToken cancellationToken)
{
const int maxRetryCount = 2;
HeartbeatInfo heartbeatInfo = null;
Exception heartbeatException = null;
for (var attempt = 1; attempt <= maxRetryCount; attempt++)
{
var connection = _connection;
try
{
if (connection == null)
{
connection = _connectionFactory.CreateConnection(_serverId, _endPoint);
// if we are cancelling, it's because the server has
// been shut down and we really don't need to wait.
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
}
heartbeatInfo = await GetHeartbeatInfoAsync(connection, cancellationToken).ConfigureAwait(false);
heartbeatException = null;
_connection = connection;
break;
}
catch (Exception ex)
{
heartbeatException = ex;
_connection = null;
if (connection != null)
{
connection.Dispose();
}
}
}
ServerDescription newDescription;
if (heartbeatInfo != null)
{
var averageRoundTripTime = _averageRoundTripTimeCalculator.AddSample(heartbeatInfo.RoundTripTime);
var averageRoundTripTimeRounded = TimeSpan.FromMilliseconds(Math.Round(averageRoundTripTime.TotalMilliseconds));
var isMasterResult = heartbeatInfo.IsMasterResult;
var buildInfoResult = heartbeatInfo.BuildInfoResult;
newDescription = _baseDescription.With(
averageRoundTripTime: averageRoundTripTimeRounded,
canonicalEndPoint: isMasterResult.Me,
electionId: isMasterResult.ElectionId,
lastWriteTimestamp: isMasterResult.LastWriteTimestamp,
logicalSessionTimeout: isMasterResult.LogicalSessionTimeout,
maxBatchCount: isMasterResult.MaxBatchCount,
maxDocumentSize: isMasterResult.MaxDocumentSize,
maxMessageSize: isMasterResult.MaxMessageSize,
replicaSetConfig: isMasterResult.GetReplicaSetConfig(),
state: ServerState.Connected,
tags: isMasterResult.Tags,
topologyVersion: isMasterResult.TopologyVersion,
type: isMasterResult.ServerType,
version: buildInfoResult.ServerVersion,
wireVersionRange: new Range <int>(isMasterResult.MinWireVersion, isMasterResult.MaxWireVersion));
}
else
{
newDescription = _baseDescription.With(lastUpdateTimestamp: DateTime.UtcNow);
}
if (heartbeatException != null)
{
var topologyVersion = default(Optional <TopologyVersion>);
if (heartbeatException is MongoCommandException heartbeatCommandException)
{
topologyVersion = TopologyVersion.FromMongoCommandException(heartbeatCommandException);
}
newDescription = newDescription.With(heartbeatException: heartbeatException, topologyVersion: topologyVersion);
}
newDescription = newDescription.With(reasonChanged: "Heartbeat", lastHeartbeatTimestamp: DateTime.UtcNow);
SetDescription(newDescription);
return(true);
}
protected VariableParam readVariable()
{
int type = _body.GetUShort();
int len = _body.GetUShort();
int blen = len - 4;
//byte[] data;
Unknown var;
switch (type)
{
case 1:
var = new HeartbeatInfo(1, "HeartbeatInfo");
break;
// 2-4 Unassigned
case 5:
var = new IPv4Address(5, "IPv4Address");
break;
case 6:
var = new IPv6Address(6, "IPv6Address");
break;
case 7:
var = new StateCookie(7, "StateCookie");
break;
case 8:
var = new UnrecognizedParameters(8, "UnrecognizedParameters");
break;
case 9:
var = new CookiePreservative(9, "CookiePreservative");
break;
// 10 Unassigned
case 11:
var = new HostNameAddress(11, "HostNameAddress");
break;
case 12:
var = new SupportedAddressTypes(12, "SupportedAddressTypes");
break;
case 13:
var = new OutgoingSSNResetRequestParameter(13, "OutgoingSSNResetRequestParameter");
break;
case 14:
var = new IncomingSSNResetRequestParameter(14, "IncomingSSNResetRequestParameter");
break;
case 15:
var = new SSNTSNResetRequestParameter(15, "SSNTSNResetRequestParameter");
break;
case 16:
var = new ReconfigurationResponseParameter(16, "ReconfigurationResponseParameter");
break;
case 17:
var = new AddOutgoingStreamsRequestParameter(17, "AddOutgoingStreamsRequestParameter");
break;
case 18:
var = new AddIncomingStreamsRequestParameter(18, "AddIncomingStreamsRequestParameter");
break;
// 19-32767 Unassigned
case 32768:
var = new Unknown(32768, "ReservedforECNCapable");
break;
case 32770:
var = new RandomParam(32770, "Random");
break;
case 32771:
var = new ChunkListParam(32771, "ChunkList");
break;
case 32772:
var = new RequestedHMACAlgorithmParameter(32772, "RequestedHMACAlgorithmParameter");
break;
case 32773:
var = new Unknown(32773, "Padding");
break;
case 32776:
var = new SupportedExtensions(32776, "SupportedExtensions");
break;
// 32777-49151 Unassigned
case 49152:
var = new ForwardTSNsupported(49152, "ForwardTSNsupported");
break;
case 49153:
var = new Unknown(49153, "AddIPAddress");
break;
case 49154:
var = new Unknown(49154, "DeleteIPAddress");
break;
case 49155:
var = new Unknown(49155, "ErrorCauseIndication");
break;
case 49156:
var = new Unknown(49156, "SetPrimaryAddress");
break;
case 49157:
var = new Unknown(49157, "SuccessIndication");
break;
case 49158:
var = new Unknown(49158, "AdaptationLayerIndication");
break;
default:
var = new Unknown(-1, "Unknown");
break;
}
try
{
var.readBody(_body, blen);
//logger.LogDebug("variable type " + var.getType() + " name " + var.getName());
}
catch (SctpPacketFormatException ex)
{
logger.LogError(ex.ToString());
}
if (_body.hasRemaining())
{
int mod = blen % 4;
if (mod != 0)
{
for (int pad = mod; pad < 4; pad++)
{
_body.GetByte();
}
}
}
return(var);
}
internal ElectionVote CastVote(Election election, Membership existingMembership, HeartbeatInfo info)
{
if (election == null || election.ElectionId == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn("ElectionManager.CastVote()", "Invalid election.");
}
return(null);
}
ElectionVote vote = new ElectionVote();
vote.RequestingNode = election.ElectionId.RequestingNode;
vote.Sourcenode = new ServerNode();
vote.Sourcenode.Name = _context.LocalAddress.IpAddress.ToString();
vote.Sourcenode.Priority = (int)LocalServerPriority;
if (existingMembership != null && (election.ElectionType == ElectionType.GeneralElections && existingMembership.Primary == null) || election.ElectionType == ElectionType.TakeoverElections)
{
if (election.RequestingServerInfo.LastOperationId > LocalServerInfo().LastOperationId)
{
vote.NodeVote = ElectionVote.Vote.yes;
}
else if (election.RequestingServerInfo.LastOperationId == LocalServerInfo().LastOperationId)
{
if (election.ElectionId.RequestingNode.Priority <= (int)LocalServerPriority)
{
vote.NodeVote = ElectionVote.Vote.yes;
}
else
{
ShardConfiguration sConfig = null;
if (_clusterConfigMgr != null)
{
sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
}
int configuredNodes = 0;
if (sConfig != null || sConfig.Servers != null)
{
configuredNodes = sConfig.Servers.Nodes.Count;
}
if (info != null && info.CSStatus == ConnectivityStatus.Connected && (configuredNodes != null && existingMembership.Servers.Count >= configuredNodes / 2))
{
vote.NodeVote = ElectionVote.Vote.no;
}
else
{
vote.NodeVote = ElectionVote.Vote.yes;
}
}
}
else
{
vote.NodeVote = ElectionVote.Vote.no;
}
}
else
{
vote.NodeVote = ElectionVote.Vote.no;
}
return(vote);
}
private void TcpConnectionEstablished(TcpPackageConnection connection)
{
if (_state != ConnectionState.Connecting || _connection != connection || connection.IsClosed)
{
LogDebug("IGNORED (_state {0}, _conn.Id {1:B}, conn.Id {2:B}, conn.closed {3}): TCP connection to [{4}, L{5}] established.",
_state, _connection == null ? Guid.Empty : _connection.ConnectionId, connection.ConnectionId,
connection.IsClosed, connection.RemoteEndPoint, connection.LocalEndPoint);
return;
}
LogDebug("TCP connection to [{0}, L{1}, {2:B}] established.", connection.RemoteEndPoint, connection.LocalEndPoint, connection.ConnectionId);
_heartbeatInfo = new HeartbeatInfo(_packageNumber, true, _stopwatch.Elapsed);
if (_settings.DefaultUserCredentials != null)
{
_connectingPhase = ConnectingPhase.Authentication;
_authInfo = new AuthInfo(Guid.NewGuid(), _stopwatch.Elapsed);
_connection.EnqueueSend(new TcpPackage(TcpCommand.Authenticate,
TcpFlags.Authenticated,
_authInfo.CorrelationId,
_settings.DefaultUserCredentials.Username,
_settings.DefaultUserCredentials.Password,
null));
}
else
{
GoToConnectedState();
}
}
/// <summary>
/// Speculative Phase:
/// 1. We search for the node with the latest op-log entry. This detail bears the highest value.
/// 2. Next, if multiple nodes have the same op-log entry, we move onto the next step.
/// 3. All those nodes which have the same op-log entry AND are connected to the CS are considered.
/// 4. Highest priority from amongst these active nodes are taken into consideration.
/// 5. If this node fulfills all of the above, it successfully passes the speculative phase.
/// </summary>
/// <param name="heartbeatReport"></param>
/// <returns></returns>
private bool ShouldIInitiateElection(LocalShardHeartbeatReporting heartbeatReport, Activity activity)
{
IList <Address> activeNodes = null;
if (heartbeatReport != null)
{
activeNodes = heartbeatReport.GetReportTable.Keys.ToList();
OperationId maxOplog = null;
IList <string> matchingOplogServerIPs = new List <string>();
HeartbeatInfo localHeartbeat = heartbeatReport.GetHeartbeatInfo(_context.LocalAddress);
if (localHeartbeat == null)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsWarnEnabled)
{
LoggerManager.Instance.ShardLogger.Warn(
"ElectionBasedMembershipStrategy.ShouldIInititateElections()",
"local node heartbeat is null");
}
return(false);
}
OperationId lastRepId = null;
if (activity.Equals(Activity.TakeoverElectionsTriggered))
{
HeartbeatInfo info = null;
if (_shard != null && _shard.Primary != null)
{
info = heartbeatReport.GetHeartbeatInfo(_shard.Primary.Address);
}
}
else
{
ShardConfiguration sConfig = _clusterConfigMgr.GetShardConfiguration(_context.LocalShardName);
int configuredNodesCount = 0;
OperationId OpIdAtCS = null;
if (sConfig != null && sConfig.Servers != null && sConfig.Servers.Nodes != null)
{
configuredNodesCount = sConfig.Servers.Nodes.Count;
}
if (configuredNodesCount > 0 && activeNodes != null && activeNodes.Count < configuredNodesCount)
{
ShardInfo sInfo = null;
ClusterInfo cInfo = _context.ConfigurationSession.GetDatabaseClusterInfo(_context.ClusterName);
if (cInfo != null)
{
sInfo = cInfo.GetShardInfo(_context.LocalShardName);
}
if (sInfo != null)
{
OpIdAtCS = sInfo.LastOperationId;
}
if (OpIdAtCS > lastRepId)
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
LoggerManager.Instance.ShardLogger.Info(
"electBasedMemSt.ShouldIInitElections()",
"CS has an operation newer than my operation. Hence, waiting.");
}
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsInfoEnabled)
{
if (OpIdAtCS != null)
{
LoggerManager.Instance.ShardLogger.Info(
"electBasedMemSt.ShouldIInitElections()",
"Operation ID on the CS:- " + OpIdAtCS.ElectionId + ":"
+ OpIdAtCS.ElectionBasedSequenceId);
}
else
{
LoggerManager.Instance.ShardLogger.Info(
"electBasedMemSt.ShouldIInitElections()", "The operation ID at the CS is set to null.");
}
if (lastRepId != null)
{
LoggerManager.Instance.ShardLogger.Info(
"electBasedMemSt.ShouldIInitElections()", "Local node Operation ID:- " + lastRepId.ElectionId +
":" + lastRepId.ElectionBasedSequenceId);
}
else
{
LoggerManager.Instance.ShardLogger.Info(
"electBasedMemSt.ShouldIInitElections()", "The local node operation ID is set to null.");
}
}
//We maintain the last replicated operation log entry with the CS.
//If a node in a shard with older data (usually the previous secondary) is up before the node with the
//latest data(usually the previous primary), it waits for a configurable amount of time (2 minutes for
//now) before proceeding with the election procedure if it is still unable to detect a primary node.
//This way we give the node with the latest data a chance to become primary and therefore avoid data loss.
lock (_mutexOnWait)
{
Monitor.Wait(_mutexOnWait, _waitTimeout);
}
}
}
for (int i = 0; i < activeNodes.Count; i++)
{
HeartbeatInfo info = heartbeatReport.GetHeartbeatInfo(activeNodes[i]);
OperationId currIndexOplog = info.LastOplogOperationId;
if (currIndexOplog > maxOplog)
{
maxOplog = currIndexOplog;
}
if (((localHeartbeat.LastOplogOperationId == null && info.LastOplogOperationId == null) || localHeartbeat.LastOplogOperationId != null && localHeartbeat.LastOplogOperationId.Equals(info.LastOplogOperationId)) &&
info.CSStatus == ConnectivityStatus.Connected)
{
matchingOplogServerIPs.Add(activeNodes[i].IpAddress.ToString());
}
}
}
if (localHeartbeat.LastOplogOperationId != null && maxOplog != null && maxOplog > localHeartbeat.LastOplogOperationId && (lastRepId == null || maxOplog > lastRepId))
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "Local operation log is behind max op log wrt " + _context.LocalShardName + " shard.");
}
if (maxOplog != null && LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "maxOplog: " + maxOplog.ElectionId + ":" + maxOplog.ElectionBasedSequenceId);
}
if (localHeartbeat.LastOplogOperationId != null && LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "local opLog (from the heartbeat): " + localHeartbeat.LastOplogOperationId.ElectionId + ":" + localHeartbeat.LastOplogOperationId.ElectionBasedSequenceId);
}
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled && lastRepId != null)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "LastOpFromOpLog (from the replication module): " + lastRepId.ElectionId + ":" + lastRepId.ElectionBasedSequenceId);
}
return(false);
}
else if (maxOplog == localHeartbeat.LastOplogOperationId || (lastRepId != null && lastRepId.Equals(maxOplog)))
{
//if: there are multiple nodes that have the same oplog entry,
//decision will be made on the basis of the priorities.
//else: the node with the highest oplog entry will be considered eligible.
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "Local operation log is equal to the max op log wrt " + _context.LocalShardName + " shard.");
}
if (maxOplog != null && (localHeartbeat != null && localHeartbeat.LastOplogOperationId != null))
{
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "maxOplog: " + maxOplog.ElectionId + ":" + maxOplog.ElectionBasedSequenceId);
}
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "local opLog (from the heartbeat): " + localHeartbeat.LastOplogOperationId.ElectionId + ":" + localHeartbeat.LastOplogOperationId.ElectionBasedSequenceId);
}
if (LoggerManager.Instance.ShardLogger != null && LoggerManager.Instance.ShardLogger.IsDebugEnabled && lastRepId != null)
{
LoggerManager.Instance.ShardLogger.Debug("electBasedMemSt.ShouldIInitiateElect()", "LastOpFromOpLog (from the replication module): " + lastRepId.ElectionId + ":" + lastRepId.ElectionBasedSequenceId);
}
}
if (matchingOplogServerIPs.Count > 0)
{
int highestRunningNodePriority = _manager.GetHighestNodePriority(activeNodes,
matchingOplogServerIPs);
if (highestRunningNodePriority.Equals(_manager.LocalServerPriority))
{
if (LoggerManager.Instance.ShardLogger != null &&
LoggerManager.Instance.ShardLogger.IsDebugEnabled)
{
LoggerManager.Instance.ShardLogger.Debug(
"ElectionBasedMembershipStrategy.ShouldIInitiateElection()",
"Node : " + _context.LocalAddress.IpAddress.ToString() + " in shard: " +
_context.LocalShardName + " is eligible having priority: " +
highestRunningNodePriority + " .");
}
return(true);
}
else
{
return(false);
}
}
}
}
return(false);
}
private void ManageHeartbeats()
{
if (_connection == null) throw new Exception();
var timeout = _heartbeatInfo.IsIntervalStage ? _settings.HeartbeatInterval : _settings.HeartbeatTimeout;
if (_stopwatch.Elapsed - _heartbeatInfo.TimeStamp < timeout)
return;
var packageNumber = _packageNumber;
if (_heartbeatInfo.LastPackageNumber != packageNumber)
{
_heartbeatInfo = new HeartbeatInfo(packageNumber, true, _stopwatch.Elapsed);
return;
}
if (_heartbeatInfo.IsIntervalStage)
{
// TcpMessage.Heartbeat analog
_connection.EnqueueSend(new TcpPackage(TcpCommand.HeartbeatRequestCommand, Guid.NewGuid(), null));
_heartbeatInfo = new HeartbeatInfo(_heartbeatInfo.LastPackageNumber, false, _stopwatch.Elapsed);
}
else
{
// TcpMessage.HeartbeatTimeout analog
var msg = string.Format("EventStoreConnection '{0}': closing TCP connection [{1}, L{2}, {3:B}] due to HEARTBEAT TIMEOUT at pkgNum {4}.",
_esConnection.ConnectionName, _connection.RemoteEndPoint, _connection.LocalEndPoint,
_connection.ConnectionId, packageNumber);
_settings.Log.Info(msg);
CloseTcpConnection(msg);
}
}
public Task Heartbeat(HeartbeatInfo heartbeat)
{
return(Task.Factory.StartNew(() => HeartbeatService.Heartbeat(heartbeat)));
}