private void Heartbeat() { int pingFailCount = 0; logger.Info("HeartBeat Thread Started."); //heart-beat thread handles its own errors. try { HeartbeatInfo info = new HeartbeatInfo(); info.Interval = _HeartbeatInterval; // init for cpu usage TimeSpan oldTime = Process.GetCurrentProcess().TotalProcessorTime; DateTime timeMeasured = DateTime.Now; TimeSpan newTime = new TimeSpan(0); //TODO: be language neutral here. how?? // init for cpu avail PerformanceCounter cpuCounter = new PerformanceCounter(); cpuCounter.ReadOnly = true; cpuCounter.CategoryName = "Processor"; cpuCounter.CounterName = "% Processor Time"; cpuCounter.InstanceName = "_Total"; while (!_stop) { // calculate usage newTime = Process.GetCurrentProcess().TotalProcessorTime; TimeSpan absUsage = newTime - oldTime; float flUsage = ((float)absUsage.Ticks / (DateTime.Now - timeMeasured).Ticks) * 100; info.PercentUsedCpuPower = (int)flUsage > 100 ? 100 : (int)flUsage; info.PercentUsedCpuPower = (int)flUsage < 0 ? 0 : (int)flUsage; timeMeasured = DateTime.Now; oldTime = newTime; try { // calculate avail info.PercentAvailCpuPower = 100 - (int)cpuCounter.NextValue() + info.PercentUsedCpuPower; info.PercentAvailCpuPower = info.PercentAvailCpuPower > 100 ? 100 : info.PercentAvailCpuPower; info.PercentAvailCpuPower = info.PercentAvailCpuPower < 0 ? 0 : info.PercentAvailCpuPower; } catch (Exception e) { logger.Debug("HeartBeat thread error: " + e.Message + Environment.NewLine + " Heartbeat continuing after error..."); } //have a seperate try...block since we try 3 times before giving up try { //send a heartbeat to the manager. _executor.Manager.Executor_Heartbeat(_executor.Credentials, _executor.Id, info); pingFailCount = 0; } catch (Exception e) { if (e is SocketException || e is System.Runtime.Remoting.RemotingException) { pingFailCount++; //we disconnect the executor if the manager cant be pinged three times if (pingFailCount >= 3) { logger.Error("Failed to contact manager " + pingFailCount + " times...", e); //the disconnect here should be started off on a seperate thread because: //disconnect itself waits for HeartBeatThread to stop. If the method call //to disconnect from HeartBeat wont return immediately, then there is a deadlock //with disconnect waiting for the HeartBeatThread to stop and the HeartBeatThread waiting //for the call to disconnect to return. //raise the event to indicate that the Executor has got disconnected. _executor.OnGotDisconnected(); } } else if (e is InvalidExecutorException) { //raise a disconnected event. //so that we will reconnect if needed. logger.Error("Invalid Executor exception : ", e); _executor.OnGotDisconnected(); } else { logger.Debug("Error during heartbeat. Continuing after error...", e); } } Thread.Sleep(_HeartbeatInterval * 1000); } } catch (ThreadAbortException) { Thread.ResetAbort(); logger.Info("HeartBeat thread aborted."); } catch (Exception e) { logger.Error("HeartBeat Exception. Heartbeat thread stopping...", e); } logger.Info("HeartBeat Thread Exited."); }
/// <summary> /// Updates the heartbeat info of an executor /// </summary> /// <param name="sc">security credentials to verify if the executor has permission to perform this operation /// (i.e HeartBeat, which is associated with the ExecuteThread permission)</param> /// <param name="executorId">executor id</param> /// <param name="info">heartbeat info</param> public void Executor_Heartbeat(SecurityCredentials sc, string executorId, HeartbeatInfo info) { AuthenticateUser(sc); EnsurePermission(sc, Permission.ExecuteThread); //kna : added this since sometimes disconnected Executors //may think they are still connected. _Executors[executorId].VerifyExists(); _Executors[executorId].HeartbeatUpdate(info); }
//----------------------------------------------------------------------------------------------- /// <summary> /// Updates the heartbeat info of an executor /// </summary> /// <param name="sc">security credentials to verify if the executor has permission to perform this operation /// (i.e HeartBeat, which is associated with the ExecuteThread permission)</param> /// <param name="executorId">executor id</param> /// <param name="info">heartbeat info</param> public void Executor_Heartbeat(SecurityCredentials sc, string executorId, HeartbeatInfo info) { AuthenticateUser(sc); EnsurePermission(sc, Permission.ExecuteThread); _Executors[executorId].HeartbeatUpdate(info); }
/// <summary> /// Updates the database with the heartbeat info of this executor /// </summary> /// <param name="info"></param> public void HeartbeatUpdate(HeartbeatInfo info) { // update ping time and other heartbeatinfo ExecutorStorageView executorStorage = ManagerStorageFactory.ManagerStorage().GetExecutor(_Id); executorStorage.PingTime = DateTime.Now; executorStorage.Connected = true; executorStorage.CpuUsage = info.PercentUsedCpuPower; executorStorage.AvailableCpu = info.PercentAvailCpuPower; executorStorage.TotalCpuUsage += info.Interval * (float)info.PercentUsedCpuPower / 100; ManagerStorageFactory.ManagerStorage().UpdateExecutor(executorStorage); }