private void Heartbeat()
{
int pingFailCount = 0;
logger.Info("HeartBeat Thread Started.");
//heart-beat thread handles its own errors.
try
{
HeartbeatInfo info = new HeartbeatInfo();
info.Interval = _HeartbeatInterval;
// init for cpu usage
TimeSpan oldTime = Process.GetCurrentProcess().TotalProcessorTime;
DateTime timeMeasured = DateTime.Now;
TimeSpan newTime = new TimeSpan(0);
//TODO: be language neutral here. how??
// init for cpu avail
PerformanceCounter cpuCounter = new PerformanceCounter();
cpuCounter.ReadOnly = true;
cpuCounter.CategoryName = "Processor";
cpuCounter.CounterName = "% Processor Time";
cpuCounter.InstanceName = "_Total";
while (!_stop)
{
// calculate usage
newTime = Process.GetCurrentProcess().TotalProcessorTime;
TimeSpan absUsage = newTime - oldTime;
float flUsage = ((float)absUsage.Ticks / (DateTime.Now - timeMeasured).Ticks) * 100;
info.PercentUsedCpuPower = (int)flUsage > 100 ? 100 : (int)flUsage;
info.PercentUsedCpuPower = (int)flUsage < 0 ? 0 : (int)flUsage;
timeMeasured = DateTime.Now;
oldTime = newTime;
try
{
// calculate avail
info.PercentAvailCpuPower = 100 - (int)cpuCounter.NextValue() + info.PercentUsedCpuPower;
info.PercentAvailCpuPower = info.PercentAvailCpuPower > 100 ? 100 : info.PercentAvailCpuPower;
info.PercentAvailCpuPower = info.PercentAvailCpuPower < 0 ? 0 : info.PercentAvailCpuPower;
}
catch (Exception e)
{
logger.Debug("HeartBeat thread error: " + e.Message + Environment.NewLine + " Heartbeat continuing after error...");
}
//have a seperate try...block since we try 3 times before giving up
try
{
//send a heartbeat to the manager.
_executor.Manager.Executor_Heartbeat(_executor.Credentials, _executor.Id, info);
pingFailCount = 0;
}
catch (Exception e)
{
if (e is SocketException || e is System.Runtime.Remoting.RemotingException)
{
pingFailCount++;
//we disconnect the executor if the manager cant be pinged three times
if (pingFailCount >= 3)
{
logger.Error("Failed to contact manager " + pingFailCount + " times...", e);
//the disconnect here should be started off on a seperate thread because:
//disconnect itself waits for HeartBeatThread to stop. If the method call
//to disconnect from HeartBeat wont return immediately, then there is a deadlock
//with disconnect waiting for the HeartBeatThread to stop and the HeartBeatThread waiting
//for the call to disconnect to return.
//raise the event to indicate that the Executor has got disconnected.
_executor.OnGotDisconnected();
}
}
else if (e is InvalidExecutorException)
{
//raise a disconnected event.
//so that we will reconnect if needed.
logger.Error("Invalid Executor exception : ", e);
_executor.OnGotDisconnected();
}
else
{
logger.Debug("Error during heartbeat. Continuing after error...", e);
}
}
Thread.Sleep(_HeartbeatInterval * 1000);
}
}
catch (ThreadAbortException)
{
Thread.ResetAbort();
logger.Info("HeartBeat thread aborted.");
}
catch (Exception e)
{
logger.Error("HeartBeat Exception. Heartbeat thread stopping...", e);
}
logger.Info("HeartBeat Thread Exited.");
}
/// <summary>
/// Updates the heartbeat info of an executor
/// </summary>
/// <param name="sc">security credentials to verify if the executor has permission to perform this operation
/// (i.e HeartBeat, which is associated with the ExecuteThread permission)</param>
/// <param name="executorId">executor id</param>
/// <param name="info">heartbeat info</param>
public void Executor_Heartbeat(SecurityCredentials sc, string executorId, HeartbeatInfo info)
{
AuthenticateUser(sc);
EnsurePermission(sc, Permission.ExecuteThread);
//kna : added this since sometimes disconnected Executors
//may think they are still connected.
_Executors[executorId].VerifyExists();
_Executors[executorId].HeartbeatUpdate(info);
}
//-----------------------------------------------------------------------------------------------
/// <summary>
/// Updates the heartbeat info of an executor
/// </summary>
/// <param name="sc">security credentials to verify if the executor has permission to perform this operation
/// (i.e HeartBeat, which is associated with the ExecuteThread permission)</param>
/// <param name="executorId">executor id</param>
/// <param name="info">heartbeat info</param>
public void Executor_Heartbeat(SecurityCredentials sc, string executorId, HeartbeatInfo info)
{
AuthenticateUser(sc);
EnsurePermission(sc, Permission.ExecuteThread);
_Executors[executorId].HeartbeatUpdate(info);
}
/// <summary>
/// Updates the database with the heartbeat info of this executor
/// </summary>
/// <param name="info"></param>
public void HeartbeatUpdate(HeartbeatInfo info)
{
// update ping time and other heartbeatinfo
ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id);
executorStorage.PingTime = DateTime.Now;
executorStorage.Connected = true;
executorStorage.CpuUsage = info.PercentUsedCpuPower;
executorStorage.AvailableCpu = info.PercentAvailCpuPower;
executorStorage.TotalCpuUsage += info.Interval * (float)info.PercentUsedCpuPower / 100;
ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage);
}