private EvaluatorHeartbeatProto GetEvaluatorHeartbeatProto()
{
return(GetEvaluatorHeartbeatProto(
EvaluatorRuntime.GetEvaluatorStatus(),
ContextManager.GetContextStatusCollection(),
ContextManager.GetTaskStatus()));
}
/// <summary>
/// Called with a specific TaskStatus that must be delivered to the driver
/// </summary>
/// <param name="taskStatusProto"></param>
public void OnNext(TaskStatusProto taskStatusProto)
{
LOGGER.Log(Level.Verbose, "Before acquiring lock: HeartbeatManager::OnNext(TaskStatusProto)");
lock (this)
{
LOGGER.Log(Level.Verbose, "HeartbeatManager::OnNext(TaskStatusProto)");
EvaluatorHeartbeatProto heartbeatProto = GetEvaluatorHeartbeatProto(
EvaluatorRuntime.GetEvaluatorStatus(),
ContextManager.GetContextStatusCollection(),
Optional <TaskStatusProto> .Of(taskStatusProto));
LOGGER.Log(Level.Info, string.Format(CultureInfo.InvariantCulture, "Triggered a heartbeat: {0}.", heartbeatProto));
Send(heartbeatProto);
}
}
public void OnNext(Alarm value)
{
LOGGER.Log(Level.Verbose, "Before acquiring lock: HeartbeatManager::OnNext(Alarm)");
lock (this)
{
LOGGER.Log(Level.Verbose, "HeartbeatManager::OnNext(Alarm)");
if (_evaluatorSettings.OperationState == EvaluatorOperationState.OPERATIONAL && EvaluatorRuntime.State == State.RUNNING)
{
try
{
EvaluatorHeartbeatProto evaluatorHeartbeatProto = GetEvaluatorHeartbeatProto();
LOGGER.Log(Level.Verbose,
string.Format(CultureInfo.InvariantCulture, "Triggered a heartbeat: {0}. {1}Node Health: {2}", evaluatorHeartbeatProto, Environment.NewLine, MachineStatus));
Send(evaluatorHeartbeatProto);
}
catch (Exception e)
{
Utilities.Diagnostics.Exceptions.Caught(e, Level.Error, LOGGER);
EvaluatorRuntime.OnException(e);
}
}
else
{
LOGGER.Log(Level.Verbose, "Ignoring regular heartbeat since Evaluator operation state is [{0}] and runtime state is [{1}]. ", EvaluatorSettings.OperationState, EvaluatorRuntime.State);
// Do not try to recover if Evaluator is done.
if (IsEvaluatorStateCompleted(EvaluatorRuntime.State))
{
return;
}
if (_evaluatorSettings.OperationState == EvaluatorOperationState.RECOVERY)
{
var driverConnection = _driverConnection.Get();
try
{
var driverInformation = driverConnection.GetDriverInformation();
if (driverInformation == null)
{
LOGGER.Log(Level.Verbose,
"In RECOVERY mode, cannot retrieve driver information, will try again later.");
}
else
{
var msg = string.Format(CultureInfo.InvariantCulture,
"Detect driver restarted at {0} and is running on endpoint {1} with services {2}. Now trying to re-establish connection",
driverInformation.DriverStartTime,
driverInformation.DriverRemoteIdentifier,
driverInformation.NameServerId);
LOGGER.Log(Level.Info, msg);
Recover(driverInformation);
}
}
catch (NotImplementedException)
{
LOGGER.Log(Level.Error, "Reaching EvaluatorOperation RECOVERY mode, however, there is no IDriverConnection implemented for HA.");
throw;
}
catch (Exception e)
{
// we do not want any exception to stop the query for driver status
Utilities.Diagnostics.Exceptions.Caught(e, Level.Warning, LOGGER);
}
}
}
_clock.ScheduleAlarm(_heartBeatPeriodInMillSeconds, this);
}
}
